From 841aabf5b383bf9da84b3c3f8ef724c9f23f2d52 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Thu, 5 Sep 2024 18:00:55 -0700 Subject: [PATCH 01/34] Implement LangGraph Scheduler for Kafka - Orchestrator and Executor classes to run LangGraph in a distributed fashion using Kafka as a message bus for communication - Orchestrator and Executor run on-demand when a new message is published to the topic they listen to - Orchestrator is responsible for running the Pregel algorithm (deciding next tasks to run) and sending messages to the executor topic - Executor is responsible for executing each task (node), and sending messages to the orchestrator topic when done --- libs/langgraph/langgraph/constants.py | 2 + libs/langgraph/langgraph/pregel/loop.py | 8 +- libs/scheduler-kafka/LICENSE | 21 + libs/scheduler-kafka/Makefile | 46 + libs/scheduler-kafka/README.md | 3 + .../langgraph/scheduler/kafka/__init__.py | 0 .../langgraph/scheduler/kafka/executor.py | 94 ++ .../langgraph/scheduler/kafka/orchestrator.py | 94 ++ .../langgraph/scheduler/kafka/serde.py | 16 + .../langgraph/scheduler/kafka/types.py | 25 + .../langgraph/scheduler/py.typed | 0 libs/scheduler-kafka/poetry.lock | 1182 +++++++++++++++++ libs/scheduler-kafka/pyproject.toml | 57 + libs/scheduler-kafka/testss/__init__.py | 0 libs/scheduler-kafka/testss/compose.yml | 20 + libs/scheduler-kafka/testss/conftest.py | 58 + libs/scheduler-kafka/testss/test_scheduler.py | 124 ++ 17 files changed, 1749 insertions(+), 1 deletion(-) create mode 100644 libs/scheduler-kafka/LICENSE create mode 100644 libs/scheduler-kafka/Makefile create mode 100644 libs/scheduler-kafka/README.md create mode 100644 libs/scheduler-kafka/langgraph/scheduler/kafka/__init__.py create mode 100644 libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py create mode 100644 libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py create mode 100644 libs/scheduler-kafka/langgraph/scheduler/kafka/serde.py create mode 100644 libs/scheduler-kafka/langgraph/scheduler/kafka/types.py create mode 100644 libs/scheduler-kafka/langgraph/scheduler/py.typed create mode 100644 libs/scheduler-kafka/poetry.lock create mode 100644 libs/scheduler-kafka/pyproject.toml create mode 100644 libs/scheduler-kafka/testss/__init__.py create mode 100644 libs/scheduler-kafka/testss/compose.yml create mode 100644 libs/scheduler-kafka/testss/conftest.py create mode 100644 libs/scheduler-kafka/testss/test_scheduler.py diff --git a/libs/langgraph/langgraph/constants.py b/libs/langgraph/langgraph/constants.py index f19c562f5..9cc287277 100644 --- a/libs/langgraph/langgraph/constants.py +++ b/libs/langgraph/langgraph/constants.py @@ -9,6 +9,7 @@ CONFIG_KEY_STREAM = "__pregel_stream" CONFIG_KEY_STORE = "__pregel_store" CONFIG_KEY_RESUMING = "__pregel_resuming" CONFIG_KEY_TASK_ID = "__pregel_task_id" +CONFIG_KEY_DEDUPE_TASKS = "__pregel_dedupe_tasks" # this one part of public API so more readable CONFIG_KEY_CHECKPOINT_MAP = "checkpoint_map" INTERRUPT = "__interrupt__" @@ -30,6 +31,7 @@ RESERVED = { CONFIG_KEY_STORE, CONFIG_KEY_RESUMING, CONFIG_KEY_TASK_ID, + CONFIG_KEY_DEDUPE_TASKS, INPUT, RUNTIME_PLACEHOLDER, } diff --git a/libs/langgraph/langgraph/pregel/loop.py b/libs/langgraph/langgraph/pregel/loop.py index afd29ec1b..c18fe5efb 100644 --- a/libs/langgraph/langgraph/pregel/loop.py +++ b/libs/langgraph/langgraph/pregel/loop.py @@ -39,6 +39,7 @@ from langgraph.checkpoint.base import ( ) from langgraph.constants import ( CONFIG_KEY_CHECKPOINT_MAP, + CONFIG_KEY_DEDUPE_TASKS, CONFIG_KEY_RESUMING, CONFIG_KEY_STREAM, CONFIG_KEY_TASK_ID, @@ -179,7 +180,10 @@ class PregelLoop: self.output_keys = output_keys self.stream_keys = stream_keys self.is_nested = CONFIG_KEY_TASK_ID in self.config.get("configurable", {}) - self.skip_done_tasks = "checkpoint_id" not in config["configurable"] + self.skip_done_tasks = ( + "checkpoint_id" not in config["configurable"] + or CONFIG_KEY_DEDUPE_TASKS in config["configurable"] + ) self.debug = debug if CONFIG_KEY_STREAM in config["configurable"]: self.stream = DuplexStream( @@ -595,6 +599,7 @@ class SyncPregelLoop(PregelLoop, ContextManager): **self.config, **saved.config, "configurable": { + "checkpoint_ns": "", **self.config.get("configurable", {}), **saved.config.get("configurable", {}), }, @@ -697,6 +702,7 @@ class AsyncPregelLoop(PregelLoop, AsyncContextManager): **self.config, **saved.config, "configurable": { + "checkpoint_ns": "", **self.config.get("configurable", {}), **saved.config.get("configurable", {}), }, diff --git a/libs/scheduler-kafka/LICENSE b/libs/scheduler-kafka/LICENSE new file mode 100644 index 000000000..fc0602fee --- /dev/null +++ b/libs/scheduler-kafka/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 LangChain, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/libs/scheduler-kafka/Makefile b/libs/scheduler-kafka/Makefile new file mode 100644 index 000000000..53c67e9a4 --- /dev/null +++ b/libs/scheduler-kafka/Makefile @@ -0,0 +1,46 @@ +.PHONY: test test_watch lint format + +###################### +# TESTING AND COVERAGE +###################### + +start-services: + docker compose -f testss/compose.yml up -V --force-recreate --wait --remove-orphans + +stop-services: + docker compose -f testss/compose.yml down + +test: + make start-services && poetry run pytest; \ + EXIT_CODE=$$?; \ + make stop-services; \ + exit $$EXIT_CODE + +test_watch: + make start-services && poetry run ptw .; \ + EXIT_CODE=$$?; \ + make stop-services; \ + exit $$EXIT_CODE + +###################### +# LINTING AND FORMATTING +###################### + +# Define a variable for Python and notebook files. +PYTHON_FILES=. +MYPY_CACHE=.mypy_cache +lint format: PYTHON_FILES=. +lint_diff format_diff: PYTHON_FILES=$(shell git diff --name-only --relative --diff-filter=d main . | grep -E '\.py$$|\.ipynb$$') +lint_package: PYTHON_FILES=langgraph +lint_tests: PYTHON_FILES=tests +lint_tests: MYPY_CACHE=.mypy_cache_test + +lint lint_diff lint_package lint_tests: + poetry run ruff check . + [ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES) --diff + [ "$(PYTHON_FILES)" = "" ] || poetry run ruff check --select I $(PYTHON_FILES) + [ "$(PYTHON_FILES)" = "" ] || mkdir -p $(MYPY_CACHE) || poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE) + +format format_diff: + poetry run ruff format $(PYTHON_FILES) + poetry run ruff check --select I --fix $(PYTHON_FILES) diff --git a/libs/scheduler-kafka/README.md b/libs/scheduler-kafka/README.md new file mode 100644 index 000000000..cedfcc53f --- /dev/null +++ b/libs/scheduler-kafka/README.md @@ -0,0 +1,3 @@ +# LangGraph Scheduler for Kafka + +... diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/__init__.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py new file mode 100644 index 000000000..de1e61e09 --- /dev/null +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -0,0 +1,94 @@ +import asyncio +from contextlib import AbstractAsyncContextManager +from typing import Any + +import aiokafka + +import langgraph.scheduler.kafka.serde as serde +from langgraph.constants import ERROR +from langgraph.errors import TaskNotFound +from langgraph.pregel import Pregel +from langgraph.pregel.algo import prepare_single_task +from langgraph.pregel.executor import AsyncBackgroundExecutor +from langgraph.pregel.manager import AsyncChannelsManager +from langgraph.pregel.runner import PregelRunner +from langgraph.scheduler.kafka.types import ( + MessageToExecutor, + MessageToOrchestrator, + Topics, +) + + +class KafkaExecutor(AbstractAsyncContextManager): + def __init__(self, graph: Pregel, topics: Topics, **kwargs: Any) -> None: + self.graph = graph + self.topics = topics + self.consumer = aiokafka.AIOKafkaConsumer( + topics.executor, value_deserializer=serde.loads, **kwargs + ) + self.producer = aiokafka.AIOKafkaProducer( + value_serializer=serde.dumps, **kwargs + ) + + async def __aenter__(self) -> "KafkaExecutor": + await self.consumer.start() + await self.producer.start() + return self + + async def __aexit__(self, *args: Any) -> None: + await self.consumer.stop() + await self.producer.stop() + + def __aiter__(self) -> "KafkaExecutor": + return self + + async def __anext__(self) -> Any: + # wait for next message + try: + rec = await self.consumer.getone() + msg: MessageToExecutor = rec.value + except aiokafka.ConsumerStoppedError: + raise StopAsyncIteration from None + # process message + saved = await self.graph.checkpointer.aget_tuple(msg["config"]) + if saved is None: + raise RuntimeError("Checkpoint not found") + async with AsyncChannelsManager( + self.graph.channels, saved.checkpoint, msg["config"], self.graph.store + ) as (channels, managed), AsyncBackgroundExecutor() as submit: + + def put_writes(task_id: str, writes: list[tuple[str, Any]]) -> None: + print("put_writes", task_id, writes) + return submit( + self.graph.checkpointer.aput_writes, msg["config"], writes, task_id + ) + + if task := await asyncio.to_thread( + prepare_single_task, + msg["task"]["path"], + msg["task"]["id"], + checkpoint=saved.checkpoint, + processes=self.graph.nodes, + channels=channels, + managed=managed, + config=msg["config"], + step=msg["task"]["step"], + for_execution=True, + is_resuming=msg["task"]["resuming"], + ): + # execute task, saving writes + runner = PregelRunner(submit=submit, put_writes=put_writes) + async for _ in runner.atick([task]): + pass + else: + # task was not found + await self.graph.checkpointer.put_writes( + msg["config"], [(ERROR, TaskNotFound())] + ) + # notify orchestrator + await self.producer.send( + self.topics.orchestrator, + value=MessageToOrchestrator(input=None, config=msg["config"]), + ) + # return message + return msg diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py new file mode 100644 index 000000000..5187f0435 --- /dev/null +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -0,0 +1,94 @@ +from contextlib import AbstractAsyncContextManager +from typing import Any + +import aiokafka +from langchain_core.runnables import ensure_config + +import langgraph.scheduler.kafka.serde as serde +from langgraph.constants import CONFIG_KEY_DEDUPE_TASKS, SCHEDULED +from langgraph.pregel import Pregel +from langgraph.pregel.loop import INPUT_RESUMING, AsyncPregelLoop +from langgraph.scheduler.kafka.types import ( + ExecutorTask, + MessageToExecutor, + MessageToOrchestrator, + Topics, +) +from langgraph.utils.config import patch_configurable + + +class KafkaOrchestrator(AbstractAsyncContextManager): + def __init__(self, graph: Pregel, topics: Topics, **kwargs: Any) -> None: + self.graph = graph + self.topics = topics + self.consumer = aiokafka.AIOKafkaConsumer( + topics.orchestrator, value_deserializer=serde.loads, **kwargs + ) + self.producer = aiokafka.AIOKafkaProducer( + value_serializer=serde.dumps, **kwargs + ) + + async def __aenter__(self) -> "KafkaOrchestrator": + await self.consumer.start() + await self.producer.start() + return self + + async def __aexit__(self, *args: Any) -> None: + await self.consumer.stop() + await self.producer.stop() + + def __aiter__(self) -> "KafkaOrchestrator": + return self + + async def __anext__(self) -> Any: + # wait for next message + try: + rec = await self.consumer.getone() + msg: MessageToOrchestrator = rec.value + except aiokafka.ConsumerStoppedError: + raise StopAsyncIteration from None + # process message + async with AsyncPregelLoop( + msg["input"], + config=ensure_config(msg["config"]), + stream=None, + store=self.graph.store, + checkpointer=self.graph.checkpointer, + nodes=self.graph.nodes, + specs=self.graph.channels, + output_keys=self.graph.output_channels, + stream_keys=self.graph.stream_channels, + ) as loop: + if loop.tick(input_keys=self.graph.input_channels): + if hasattr(loop, "_put_checkpoint_fut"): + await loop._put_checkpoint_fut + if new_tasks := [t for t in loop.tasks.values() if not t.scheduled]: + # send messages to executor + for task in new_tasks: + if task.scheduled: + continue + await self.producer.send( + self.topics.executor, + value=MessageToExecutor( + config=patch_configurable( + loop.config, + { + **loop.checkpoint_config["configurable"], + CONFIG_KEY_DEDUPE_TASKS: True, + }, + ), + task=ExecutorTask( + id=task.id, + path=task.path, + step=loop.step, + resuming=loop.input is INPUT_RESUMING, + ), + ), + ) + # flush producer + await self.producer.flush() + # mark as scheduled + for task in new_tasks: + loop.put_writes(task.id, [(SCHEDULED, None)]) + # return message + return msg diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/serde.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/serde.py new file mode 100644 index 000000000..952de1721 --- /dev/null +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/serde.py @@ -0,0 +1,16 @@ +from typing import Any + +import orjson + + +def loads(v: bytes) -> Any: + return orjson.loads(v) + + +def dumps(v: Any) -> bytes: + return orjson.dumps(v, default=_default) + + +def _default(v: Any) -> Any: + # things we don't know how to serialize (eg. functions) ignore + return None diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py new file mode 100644 index 000000000..ce2a01d9b --- /dev/null +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py @@ -0,0 +1,25 @@ +from typing import Any, NamedTuple, Optional, TypedDict + +from langchain_core.runnables import RunnableConfig + + +class Topics(NamedTuple): + orchestrator: str + executor: str + + +class MessageToOrchestrator(TypedDict): + input: Optional[dict[str, Any]] + config: RunnableConfig + + +class ExecutorTask(TypedDict): + id: str + path: tuple[str, ...] + step: int + resuming: bool + + +class MessageToExecutor(TypedDict): + config: RunnableConfig + task: ExecutorTask diff --git a/libs/scheduler-kafka/langgraph/scheduler/py.typed b/libs/scheduler-kafka/langgraph/scheduler/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/libs/scheduler-kafka/poetry.lock b/libs/scheduler-kafka/poetry.lock new file mode 100644 index 000000000..14ac0aac5 --- /dev/null +++ b/libs/scheduler-kafka/poetry.lock @@ -0,0 +1,1182 @@ +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "aiokafka" +version = "0.11.0" +description = "Kafka integration with asyncio" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiokafka-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:926f93fb6a39891fd4364494432b479c0602f9cac708778d4a262a2c2e20d3b4"}, + {file = "aiokafka-0.11.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38e1917e706c1158d5e1f612d1fc1b40f706dc46c534e73ab4de8ae2868a31be"}, + {file = "aiokafka-0.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:516e1d68d9a377860b2e17453580afe304605bc71894f684d3e7b6618f6f939f"}, + {file = "aiokafka-0.11.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acfd0a5d0aec762ba73eeab73b23edce14f315793f063b6a4b223b6f79e36bb8"}, + {file = "aiokafka-0.11.0-cp310-cp310-win32.whl", hash = "sha256:0d80590c4ef0ba546a299cee22ea27c3360c14241ec43a8e6904653f7b22d328"}, + {file = "aiokafka-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:1d519bf9875ac867fb19d55de3750833b1eb6379a08de29a68618e24e6a49fc0"}, + {file = "aiokafka-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0e957b42ae959365efbb45c9b5de38032c573608553c3670ad8695cc210abec9"}, + {file = "aiokafka-0.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:224db2447f6c1024198d8342e7099198f90401e2fa29c0762afbc51eadf5c490"}, + {file = "aiokafka-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef3e7c8a923e502caa4d24041f2be778fd7f9ee4587bf0bcb4f74cac05122fa"}, + {file = "aiokafka-0.11.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:59f4b935589ebb244620afad8bf3320e3bc86879a8b1c692ad06bd324f6c6127"}, + {file = "aiokafka-0.11.0-cp311-cp311-win32.whl", hash = "sha256:560839ae6bc13e71025d71e94df36980f5c6e36a64916439e598b6457267a37f"}, + {file = "aiokafka-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:1f8ae91f0373830e4664376157fe61b611ca7e573d8a559b151aef5bf53df46c"}, + {file = "aiokafka-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4e0cc080a7f4c659ee4e1baa1c32adedcccb105a52156d4909f357d76fac0dc1"}, + {file = "aiokafka-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55a07a39d82c595223a17015ea738d152544cee979d3d6d822707a082465621c"}, + {file = "aiokafka-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3711fa64ee8640dcd4cb640f1030f9439d02e85acd57010d09053017092d8cc2"}, + {file = "aiokafka-0.11.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:818a6f8e44b02113b9e795bee6029c8a4e525ab38f29d7adb0201f3fec74c808"}, + {file = "aiokafka-0.11.0-cp312-cp312-win32.whl", hash = "sha256:8ba981956243767b37c929845c398fda2a2e35a4034d218badbe2b62e6f98f96"}, + {file = "aiokafka-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a478a14fd23fd1ffe9c7a21238d818b5f5e0626f7f06146b687f3699298391b"}, + {file = "aiokafka-0.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0973a245b8b9daf8ef6814253a80a700f1f54d2da7d88f6fe479f46e0fd83053"}, + {file = "aiokafka-0.11.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee0c61a2dcabbe4474ff237d708f9bd663dd2317e03a9cb7239a212c9ee05b12"}, + {file = "aiokafka-0.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:230170ce2e8a0eb852e2e8b78b08ce2e29b77dfe2c51bd56f5ab4be0f332a63b"}, + {file = "aiokafka-0.11.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eac78a009b713e28b5b4c4daae9d062acbf2b7980e5734467643a810134583b5"}, + {file = "aiokafka-0.11.0-cp38-cp38-win32.whl", hash = "sha256:73584be8ba7906e3f33ca0f08f6af21a9ae31b86c6b635b93db3b1e6f452657b"}, + {file = "aiokafka-0.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:d724b6fc484e453b373052813e4e543fc028a22c3fbda10e13b6829740000b8a"}, + {file = "aiokafka-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:419dd28c8ed6e926061bdc60929af08a6b52f1721e1179d9d21cc72ae28fd6f6"}, + {file = "aiokafka-0.11.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1c85f66eb3564c5e74d8e4c25df4ac1fd94f1a6f6e66f005aafa6f791bde215"}, + {file = "aiokafka-0.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaafe134de57b184f3c030e1a11051590caff7953c8bf58048eefd8d828e39d7"}, + {file = "aiokafka-0.11.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:807f699cf916369b1a512e4f2eaec714398c202d8803328ef8711967d99a56ce"}, + {file = "aiokafka-0.11.0-cp39-cp39-win32.whl", hash = "sha256:d59fc7aec088c9ffc02d37e61591f053459bd11912cf04c70ac4f7e60405667d"}, + {file = "aiokafka-0.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:702aec15b63bad5e4476294bcb1cb177559149fce3e59335794f004c279cbd6a"}, + {file = "aiokafka-0.11.0.tar.gz", hash = "sha256:f2def07fe1720c4fe37c0309e355afa9ff4a28e0aabfe847be0692461ac69352"}, +] + +[package.dependencies] +async-timeout = "*" +packaging = "*" +typing-extensions = ">=4.10.0" + +[package.extras] +all = ["cramjam (>=2.8.0)", "gssapi"] +gssapi = ["gssapi"] +lz4 = ["cramjam (>=2.8.0)"] +snappy = ["cramjam"] +zstd = ["cramjam"] + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "anyio" +version = "4.4.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"}, + {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + +[[package]] +name = "async-timeout" +version = "4.0.3" +description = "Timeout context manager for asyncio programs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, +] + +[[package]] +name = "certifi" +version = "2024.8.30" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "codespell" +version = "2.3.0" +description = "Codespell" +optional = false +python-versions = ">=3.8" +files = [ + {file = "codespell-2.3.0-py3-none-any.whl", hash = "sha256:a9c7cef2501c9cfede2110fd6d4e5e62296920efe9abfb84648df866e47f58d1"}, + {file = "codespell-2.3.0.tar.gz", hash = "sha256:360c7d10f75e65f67bad720af7007e1060a5d395670ec11a7ed1fed9dd17471f"}, +] + +[package.extras] +dev = ["Pygments", "build", "chardet", "pre-commit", "pytest", "pytest-cov", "pytest-dependency", "ruff", "tomli", "twine"] +hard-encoding-detection = ["chardet"] +toml = ["tomli"] +types = ["chardet (>=5.1.0)", "mypy", "pytest", "pytest-cov", "pytest-dependency"] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "crc32c" +version = "2.7.post1" +description = "A python package implementing the crc32c algorithm in hardware and software" +optional = false +python-versions = ">=3.7" +files = [ + {file = "crc32c-2.7.post1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c48ff8e53243839c75a5563f6cf82f60285307d58c79142a51854666f6ff0b51"}, + {file = "crc32c-2.7.post1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:60ddb274271b94e96bb3d906f5d7d6780f79365071333eac5552e6cc79934b14"}, + {file = "crc32c-2.7.post1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:375f02c2555d733092af02b942e56bc86277ff95c9dc5e1e1cb7573a9b03d11f"}, + {file = "crc32c-2.7.post1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5ed125c3add2abcbeeb37c6162451e3367e30ee328bfd7f43b4d048577b4242"}, + {file = "crc32c-2.7.post1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ebd725aef659604f1be1e3d8b36ae58740adc1d3aee61e3794ae153c10aefeca"}, + {file = "crc32c-2.7.post1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fcf277b5277a458c70c5777d1adae58eb772fe3afb5418b8bb43d2cca059f18"}, + {file = "crc32c-2.7.post1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00ce24782d166115ab8408c2e055028cf03a20a807a14aba51d87214561b699"}, + {file = "crc32c-2.7.post1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3147afad2134145d77b39c1b0add0f47fbbe6f843032ed72d0ee5d87cf31a8b9"}, + {file = "crc32c-2.7.post1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5cd10562a9715d0937ee680f9ef01f5e3ee6a3850113be1713234b3bac5d2348"}, + {file = "crc32c-2.7.post1-cp310-cp310-win32.whl", hash = "sha256:c38e981631c5063ae33cc07bec3edc47ed9afb831231f26f90b2133d90cb29ed"}, + {file = "crc32c-2.7.post1-cp310-cp310-win_amd64.whl", hash = "sha256:d5be25568c52a92b5b9ac4030935c591bc00020bed48f1b36bbffbc8382705d9"}, + {file = "crc32c-2.7.post1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1aef7771e50e9863fc642ed00b6767f045d1ad227f13e7c7f0fd14f45e597e88"}, + {file = "crc32c-2.7.post1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3e3a5381bfd8ff447639fd2fbc31eea392a1c79a8f3a67f3a9df4fb7c198593c"}, + {file = "crc32c-2.7.post1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bbb6bcddc076a95cebc578b23e28a4bef33908c5a37b11f932533baa3d873c04"}, + {file = "crc32c-2.7.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e1d3f3edb8a4dd04a2eba705a2defb646a1998ac9b4ae821239af8525385364"}, + {file = "crc32c-2.7.post1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fad064f194c65e291a056ed5e5d1a4e6d2a6fbaf10e618926295f4ec0b45c335"}, + {file = "crc32c-2.7.post1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c2e90f8832a2faa8b3dc1426fb1229de0834e76f8ebcea0329822a0d390cea0"}, + {file = "crc32c-2.7.post1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0b555694a1db21bcfa87ee9629d259dc92600b078cec8cd35c89ad2b10d397b9"}, + {file = "crc32c-2.7.post1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:78a833e3d923b781179cfcbde6be63d5d5a40ffcae592141447abedcb26f482b"}, + {file = "crc32c-2.7.post1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:328071b582c245b4653dad1671900fb3f9408eba4a42aabaefc975dabfcb6b1b"}, + {file = "crc32c-2.7.post1-cp311-cp311-win32.whl", hash = "sha256:51866ea030fd74a380cc77ed44edb48b9335b449d02bf719236e9b96a27010a5"}, + {file = "crc32c-2.7.post1-cp311-cp311-win_amd64.whl", hash = "sha256:cfb02ddb24246ec604ccf65fc8cb16620ed355a1d88725b32a2baff255924290"}, + {file = "crc32c-2.7.post1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:5ea2ce9b93cdca9c6aaa36a4fbbc963104007a81b9c8e9a0880e22967f1f09c8"}, + {file = "crc32c-2.7.post1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:2739837f83a96230b2bff39153b1f307eecb1f49c8414d1b0f5f62b6974c4721"}, + {file = "crc32c-2.7.post1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4662cacdf0094056837a517869e69aa5b1f1347b93eb30da92619f779d6e4d10"}, + {file = "crc32c-2.7.post1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b098b018cf9d5cdab845f03225fee8f345140b4a5bc09f220405368ec4772443"}, + {file = "crc32c-2.7.post1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5d1befa6ab8adab06829bb104d22e55513e852dbd8bae2d6d57721de8ab25e55"}, + {file = "crc32c-2.7.post1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa676cec7b5a4840a2639f189a003c2664ae19d6540520fccadcd89a1ae67483"}, + {file = "crc32c-2.7.post1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9543f35e87f850039f8270c9832393f22358b85be56537311ec59f038f89e1a6"}, + {file = "crc32c-2.7.post1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:017b745d622ae11ce7adf872bd25033dd3d774ed85f33d1f5fa6e98ed04ee35b"}, + {file = "crc32c-2.7.post1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0f485cb6f5efb0b50fa156a6ace781ae5c5fdb1f8b337a5af9b1231ac822bc60"}, + {file = "crc32c-2.7.post1-cp312-cp312-win32.whl", hash = "sha256:8ba922650aac153b5f52bd76ff611ac0543b4a5eb6cfd865b0e1e3d6d62fde59"}, + {file = "crc32c-2.7.post1-cp312-cp312-win_amd64.whl", hash = "sha256:8da1aec4f285b5beec53dc996af8a4274aa2419bd0e3e892cd13d93b12c389f2"}, + {file = "crc32c-2.7.post1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:76a4d48ae0a973f6cfd8e4a4d6d9232bf52b40df199bb7e0905060cc62dc93aa"}, + {file = "crc32c-2.7.post1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:acc52cc9736b4fba2b55f2c471cae21a1a9449736c553ef2423d16f502fd3076"}, + {file = "crc32c-2.7.post1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:930b8b18baa367f0924c74deedb2eab14f0d87560ce2887019cb8eebffc72537"}, + {file = "crc32c-2.7.post1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc35e5df54eae762b915007a0c247a5fb1a26e3a4bcdcd17942d2bdc52917576"}, + {file = "crc32c-2.7.post1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a300cd431fca617ebc7d96f401c6cf7ef9f3e1caa1edf0af2f93eee864f1cae5"}, + {file = "crc32c-2.7.post1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7d12bbfa9001414892cc2bead57a2c1c7b2096de51e0517dc69720d49c4dd18"}, + {file = "crc32c-2.7.post1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1b0d16f5f441eea70529c7b5354c3e6736e4b41b3ee23167d7dc149fc9827dd0"}, + {file = "crc32c-2.7.post1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:a72af86290a77c374a70ba66538710154ea2d785f1f92df803d61a2d2e26fcb9"}, + {file = "crc32c-2.7.post1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c69dc0af50f880cc21a75870c066feb41170ad4ebf57a61c6bd267fc09409113"}, + {file = "crc32c-2.7.post1-cp313-cp313-win32.whl", hash = "sha256:3d72285ba0ec2b617535a3fb1891df63c66bd23afcfe48f49c37571d494c8e80"}, + {file = "crc32c-2.7.post1-cp313-cp313-win_amd64.whl", hash = "sha256:cf029cafc0f4dc4909657630c2a9e0f64515a00d3ed2ba8e8680230aba750d32"}, + {file = "crc32c-2.7.post1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a5b328c2e0c7aba774a5cf6b5abecab6a44da2fb65ad054cfbecfbd9d56bd919"}, + {file = "crc32c-2.7.post1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:707901882f983f532a59f0dfe3188311c4c259c5184030a798669ea0a3c6df02"}, + {file = "crc32c-2.7.post1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a57ab75a795d2623cfdecfabd228415512f08436bff094f0b8d58cd99d00749d"}, + {file = "crc32c-2.7.post1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab1510d64207deeb4d984bd1fbfddef4aaeb690c895d8319093974a2acbe1eb5"}, + {file = "crc32c-2.7.post1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6334e737fc6fc686b468cb8ee04c77612bd3c5134034d4ca43f11feba4aac38"}, + {file = "crc32c-2.7.post1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cb4ed6e317f63205007a85f6b1c47ce4d7017ec79218d76544ad5bcffd86471"}, + {file = "crc32c-2.7.post1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:3919b77691c7abb53111af5fda909ea3411de54547823098bb178792baca8d8e"}, + {file = "crc32c-2.7.post1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:56baff3205a4d7b7997ded7b75a25a8999093e2be8ff007ae473da9b5d6820d3"}, + {file = "crc32c-2.7.post1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:69a40ac191cb68a0feb18f9b9c27ebce9eed1ff24b4a43bede507da58ab12bd7"}, + {file = "crc32c-2.7.post1-cp313-cp313t-win32.whl", hash = "sha256:ac23976fb8ff8d80ce9fa5f1674226b9bda5ee0724ff32629b90c959e3889e1b"}, + {file = "crc32c-2.7.post1-cp313-cp313t-win_amd64.whl", hash = "sha256:e034c90baae74e20760dede028986fb90356209b16fa5bc6b67e275009c7633e"}, + {file = "crc32c-2.7.post1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:88326cc81a4ffb043118d7d14226746264a70f2915f616818c4c447797728811"}, + {file = "crc32c-2.7.post1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c52d6fb05713e0b7ffe527e7c82fb53f53ad2cbc75ccae6daa3c6b05672b10f5"}, + {file = "crc32c-2.7.post1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b43ec3705d555deefc6cf4585c261f43e6398866b4c0ecf31bfd6fd7ae7b8f4d"}, + {file = "crc32c-2.7.post1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e1e2a143aee338800037fc690d73e50fa48d62a502b91323025d78bdcfb9b8d"}, + {file = "crc32c-2.7.post1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b98c35b65f94a6412cca8008acc56d4de9c142690eda582a6e6a56f4a5705336"}, + {file = "crc32c-2.7.post1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:6b046eb3bbe08f575ca16ed9c2ca3f1ad80c027ed26589edcb01afea46609263"}, + {file = "crc32c-2.7.post1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:24279b85bff8afbddf843f0b1db1c558e561ffc81506b3efefc848cc73c3ab74"}, + {file = "crc32c-2.7.post1-cp37-cp37m-win32.whl", hash = "sha256:fbdb75248183dbeb7d20ad367d22984091486a1c6dde005f3a27155e6922ae54"}, + {file = "crc32c-2.7.post1-cp37-cp37m-win_amd64.whl", hash = "sha256:ad25a9902c9fdfbd6398626f38d042cadbb8cf2356179d41ca973a91b6c21cde"}, + {file = "crc32c-2.7.post1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:04ed7760775cd3bc4b0b12c2709e0754718d384e8e2c071906dd3bdc951c3e29"}, + {file = "crc32c-2.7.post1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a652fae8937e87a1f407adc26059e02c2d3f55cf6054a1dac4c5493b10de4bf7"}, + {file = "crc32c-2.7.post1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:88e33b3626c3e41aad95ff5ff4a58776f4bdac47162326fda7e666022ae964a1"}, + {file = "crc32c-2.7.post1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33df9ded1059ba6400c96250e382d46ebc801b900ab570ee65501a89d0e80de9"}, + {file = "crc32c-2.7.post1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7627b798f43b474768bbaaa9aa90b527bc900486e101d11ec951b030d92b10bc"}, + {file = "crc32c-2.7.post1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a18ce07651c8aa4269f92c47db82ab706e7280fc13780996a572463b2b80c88c"}, + {file = "crc32c-2.7.post1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:43c30e2bc58b30829bee585aec679a1c51f8d6694dc1e80bca02402e0a666c3e"}, + {file = "crc32c-2.7.post1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ef825c6d1d38313856bcaf525d53a74d8a5c46dd486debef4683cbc2749c6503"}, + {file = "crc32c-2.7.post1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2ee908418a4a4bac14b401324583e76338cd42ecb3dce542f2605a2d424aa986"}, + {file = "crc32c-2.7.post1-cp38-cp38-win32.whl", hash = "sha256:ced5d7f244376eb93cd982f91086aa23fed0e35063f23f53a584f295c76c0175"}, + {file = "crc32c-2.7.post1-cp38-cp38-win_amd64.whl", hash = "sha256:720bb1fdb95c40844c210c9ddbc2126599072973e99361d78312991332e9d70d"}, + {file = "crc32c-2.7.post1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f447d50006ac554fd826ebde05df35111cc8b40278a5e024ebd3235c3992c153"}, + {file = "crc32c-2.7.post1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c543c454610ef6408b7cd58a9bc903aab93940e56d5487d516cb8f2045f489e5"}, + {file = "crc32c-2.7.post1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:560b3106b8cee56a4021688dd2d355367856e7bb2c6202df47945e616b6d11d9"}, + {file = "crc32c-2.7.post1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f54603d75db46dd4f356790cb22c5c916f34d78f831056005cdd82a162b8fb0f"}, + {file = "crc32c-2.7.post1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f2e85bbe0fd665af8dfbed561a29d5f722d4a6b332048675131ca8aed1174e5"}, + {file = "crc32c-2.7.post1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a37f866a54e4d953c122324a4480a226e9c86287f79fdbb4e925e29dfb69ed4f"}, + {file = "crc32c-2.7.post1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dea502de4416a98555bef2bdde35dad84dba9420dec61c8a8dc7c5ba24f695bd"}, + {file = "crc32c-2.7.post1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:ef67964859fdf9eff810189584f1bdfb6fb8795a64cc7df3f8ed554262886fe3"}, + {file = "crc32c-2.7.post1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c705fff86da8f99afdf2e7956671b04b1cbd895377a60f1f2e6323feea34e7ff"}, + {file = "crc32c-2.7.post1-cp39-cp39-win32.whl", hash = "sha256:5e4e5be1dd9c9f9db6353b902b4028661a3ecd70e066012b75edc34f5f800b9a"}, + {file = "crc32c-2.7.post1-cp39-cp39-win_amd64.whl", hash = "sha256:6ac6436df05f7ed2ac781d87842857ced99d385da303c8357f10e0731384c6c3"}, + {file = "crc32c-2.7.post1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:6e17edb22e7b79ec1c5637cda952efe331c0e4b461d36cee6a9c88627f191116"}, + {file = "crc32c-2.7.post1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:565da96247966333ad4c61e23b2d6515bc6c9b903f6adba1e02f2236c45becda"}, + {file = "crc32c-2.7.post1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a31c813184a858fc808b47f2300b5d10dc042b47a5d19321f6a0af63f1758cf7"}, + {file = "crc32c-2.7.post1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e889d50d3cf6d7aa5e3e21574eb31430f41587ee9d625fd47fb079f544f28eb6"}, + {file = "crc32c-2.7.post1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:fea80119cd48e6081585313527770d31520031193c2c618f3cd1a956febbed3c"}, + {file = "crc32c-2.7.post1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8e9ab5f5b34c31064debf761dd794b2584236b4aad963c6f14888027bd15b754"}, + {file = "crc32c-2.7.post1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62d32f74e3314d81a8fa1aedc2a5e11e8827be6fe2c4491e013fb76a0b1a376a"}, + {file = "crc32c-2.7.post1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d8ea8249effc834a863b8eac39ddaedd65dc6cc7ce42b8458e02d7f2aaaf8f7"}, + {file = "crc32c-2.7.post1-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd31b3192bb5f8a6b3f4fea3acb45ee1c35b0440292b23ad9fb32dae3b73b595"}, + {file = "crc32c-2.7.post1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:c0f66f76bb3674d7698ec36cf23d843bedda1149dcf884c22ba4499bf2d199d1"}, + {file = "crc32c-2.7.post1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c60a25729068d6d0153e31b52f3575125d63a1beb0a86e68392ce5e71147b3c1"}, + {file = "crc32c-2.7.post1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60deb9566de2336a1bf84c1f4569f30d2aae29176d63fe125479df81dd154aff"}, + {file = "crc32c-2.7.post1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b392645b57f8db8f45d447b37554b5feea09d87b4cb0d9ba5ae6632efaf4f204"}, + {file = "crc32c-2.7.post1-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b00926c31de9f5bf94725bf0c9494d4fb67a86a59abf061e3ec46f751d7071d"}, + {file = "crc32c-2.7.post1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d01673c3b9f839e2a86d64ea6765b580fe99210b14b7cd1c75b4d8886d154f80"}, + {file = "crc32c-2.7.post1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4cc12415c7370457ca102e712642160f718df4d42def8fd16839d203ad155d85"}, + {file = "crc32c-2.7.post1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a122287d19c52d34e050ffa099c139deb592452fed69dc1b64e1832cc7c9eb48"}, + {file = "crc32c-2.7.post1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4002200c2524eb828907d5d581156cebff3bca8798fb995ad3869293fda18ff6"}, + {file = "crc32c-2.7.post1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ffebfe6f5f2514bf284fb9abfa1970096e580b113b8ec2b93f2a65c13769d36"}, + {file = "crc32c-2.7.post1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7a6224f895b4bcf115471c211d6f4c1e8f17eca1aaf1fb4b2bf65e61c0038eee"}, + {file = "crc32c-2.7.post1.tar.gz", hash = "sha256:645aa2738710f42b9f33352ebd79bf0de6b7f9e715642d93a939b5b57452871d"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.5" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, + {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.26.0)"] + +[[package]] +name = "httpx" +version = "0.27.2" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "idna" +version = "3.8" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +files = [ + {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, + {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpointer" +version = "3.0.0" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jsonpointer-3.0.0-py2.py3-none-any.whl", hash = "sha256:13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942"}, + {file = "jsonpointer-3.0.0.tar.gz", hash = "sha256:2b2d729f2091522d61c3b31f82e11870f60b68f43fbc705cb76bf4b832af59ef"}, +] + +[[package]] +name = "kafka-python-ng" +version = "2.2.2" +description = "Pure Python client for Apache Kafka" +optional = false +python-versions = ">=3.8" +files = [ + {file = "kafka-python-ng-2.2.2.tar.gz", hash = "sha256:87ad3a766e2c0bec71d9b99bdd9e9c5cda62d96cfda61a8ca16510484d6ad7d4"}, + {file = "kafka_python_ng-2.2.2-py2.py3-none-any.whl", hash = "sha256:3fab1a03133fade1b6fd5367ff726d980e59031c4aaca9bf02c516840a4f8406"}, +] + +[package.extras] +boto = ["botocore"] +crc32c = ["crc32c"] +lz4 = ["lz4"] +snappy = ["python-snappy"] +zstd = ["zstandard"] + +[[package]] +name = "langchain-core" +version = "0.2.38" +description = "Building applications with LLMs through composability" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langchain_core-0.2.38-py3-none-any.whl", hash = "sha256:8a5729bc7e68b4af089af20eff44fe4e7ca21d0e0c87ec21cef7621981fd1a4a"}, + {file = "langchain_core-0.2.38.tar.gz", hash = "sha256:eb69dbedd344f2ee1f15bcea6c71a05884b867588fadc42d04632e727c1238f3"}, +] + +[package.dependencies] +jsonpatch = ">=1.33,<2.0" +langsmith = ">=0.1.75,<0.2.0" +packaging = ">=23.2,<25" +pydantic = [ + {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""}, + {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""}, +] +PyYAML = ">=5.3" +tenacity = ">=8.1.0,<8.4.0 || >8.4.0,<9.0.0" +typing-extensions = ">=4.7" + +[[package]] +name = "langgraph" +version = "0.2.17" +description = "Building stateful, multi-actor applications with LLMs" +optional = false +python-versions = ">=3.9.0,<4.0" +files = [] +develop = true + +[package.dependencies] +langchain-core = ">=0.2.38,<0.4" +langgraph-checkpoint = "^1.0.2" + +[package.source] +type = "directory" +url = "../langgraph" + +[[package]] +name = "langgraph-checkpoint" +version = "1.0.9" +description = "Library with base interfaces for LangGraph checkpoint savers." +optional = false +python-versions = "^3.9.0,<4.0" +files = [] +develop = true + +[package.dependencies] +langchain-core = ">=0.2.38,<0.4" + +[package.source] +type = "directory" +url = "../checkpoint" + +[[package]] +name = "langgraph-checkpoint-postgres" +version = "1.0.6" +description = "Library with a Postgres implementation of LangGraph checkpoint saver." +optional = false +python-versions = "^3.9.0,<4.0" +files = [] +develop = true + +[package.dependencies] +langgraph-checkpoint = "^1.0.8" +orjson = ">=3.10.1" +psycopg = "^3.0.0" +psycopg-pool = "^3.0.0" + +[package.source] +type = "directory" +url = "../checkpoint-postgres" + +[[package]] +name = "langsmith" +version = "0.1.115" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.115-py3-none-any.whl", hash = "sha256:04e35cfd4c2d4ff1ea10bb577ff43957b05ebb3d9eb4e06e200701f4a2b4ac9f"}, + {file = "langsmith-0.1.115.tar.gz", hash = "sha256:3b775377d858d32354f3ee0dd1ed637068cfe9a1f13e7b3bfa82db1615cdffc9"}, +] + +[package.dependencies] +httpx = ">=0.23.0,<1" +orjson = ">=3.9.14,<4.0.0" +pydantic = [ + {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""}, + {version = ">=2.7.4,<3.0.0", markers = "python_full_version >= \"3.12.4\""}, +] +requests = ">=2,<3" + +[[package]] +name = "mypy" +version = "1.11.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3824187c99b893f90c845bab405a585d1ced4ff55421fdf5c84cb7710995229"}, + {file = "mypy-1.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:96f8dbc2c85046c81bcddc246232d500ad729cb720da4e20fce3b542cab91287"}, + {file = "mypy-1.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a5d8d8dd8613a3e2be3eae829ee891b6b2de6302f24766ff06cb2875f5be9c6"}, + {file = "mypy-1.11.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:72596a79bbfb195fd41405cffa18210af3811beb91ff946dbcb7368240eed6be"}, + {file = "mypy-1.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:35ce88b8ed3a759634cb4eb646d002c4cef0a38f20565ee82b5023558eb90c00"}, + {file = "mypy-1.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:98790025861cb2c3db8c2f5ad10fc8c336ed2a55f4daf1b8b3f877826b6ff2eb"}, + {file = "mypy-1.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25bcfa75b9b5a5f8d67147a54ea97ed63a653995a82798221cca2a315c0238c1"}, + {file = "mypy-1.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bea2a0e71c2a375c9fa0ede3d98324214d67b3cbbfcbd55ac8f750f85a414e3"}, + {file = "mypy-1.11.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2b3d36baac48e40e3064d2901f2fbd2a2d6880ec6ce6358825c85031d7c0d4d"}, + {file = "mypy-1.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:d8e2e43977f0e09f149ea69fd0556623919f816764e26d74da0c8a7b48f3e18a"}, + {file = "mypy-1.11.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:1d44c1e44a8be986b54b09f15f2c1a66368eb43861b4e82573026e04c48a9e20"}, + {file = "mypy-1.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cea3d0fb69637944dd321f41bc896e11d0fb0b0aa531d887a6da70f6e7473aba"}, + {file = "mypy-1.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a83ec98ae12d51c252be61521aa5731f5512231d0b738b4cb2498344f0b840cd"}, + {file = "mypy-1.11.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c7b73a856522417beb78e0fb6d33ef89474e7a622db2653bc1285af36e2e3e3d"}, + {file = "mypy-1.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:f2268d9fcd9686b61ab64f077be7ffbc6fbcdfb4103e5dd0cc5eaab53a8886c2"}, + {file = "mypy-1.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:940bfff7283c267ae6522ef926a7887305945f716a7704d3344d6d07f02df850"}, + {file = "mypy-1.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:14f9294528b5f5cf96c721f231c9f5b2733164e02c1c018ed1a0eff8a18005ac"}, + {file = "mypy-1.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7b54c27783991399046837df5c7c9d325d921394757d09dbcbf96aee4649fe9"}, + {file = "mypy-1.11.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:65f190a6349dec29c8d1a1cd4aa71284177aee5949e0502e6379b42873eddbe7"}, + {file = "mypy-1.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:dbe286303241fea8c2ea5466f6e0e6a046a135a7e7609167b07fd4e7baf151bf"}, + {file = "mypy-1.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:104e9c1620c2675420abd1f6c44bab7dd33cc85aea751c985006e83dcd001095"}, + {file = "mypy-1.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f006e955718ecd8d159cee9932b64fba8f86ee6f7728ca3ac66c3a54b0062abe"}, + {file = "mypy-1.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:becc9111ca572b04e7e77131bc708480cc88a911adf3d0239f974c034b78085c"}, + {file = "mypy-1.11.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6801319fe76c3f3a3833f2b5af7bd2c17bb93c00026a2a1b924e6762f5b19e13"}, + {file = "mypy-1.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:c1a184c64521dc549324ec6ef7cbaa6b351912be9cb5edb803c2808a0d7e85ac"}, + {file = "mypy-1.11.0-py3-none-any.whl", hash = "sha256:56913ec8c7638b0091ef4da6fcc9136896914a9d60d54670a75880c3e5b99ace"}, + {file = "mypy-1.11.0.tar.gz", hash = "sha256:93743608c7348772fdc717af4aeee1997293a1ad04bc0ea6efa15bf65385c538"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "orjson" +version = "3.10.7" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.7-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:74f4544f5a6405b90da8ea724d15ac9c36da4d72a738c64685003337401f5c12"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34a566f22c28222b08875b18b0dfbf8a947e69df21a9ed5c51a6bf91cfb944ac"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf6ba8ebc8ef5792e2337fb0419f8009729335bb400ece005606336b7fd7bab7"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac7cf6222b29fbda9e3a472b41e6a5538b48f2c8f99261eecd60aafbdb60690c"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de817e2f5fc75a9e7dd350c4b0f54617b280e26d1631811a43e7e968fa71e3e9"}, + {file = "orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:348bdd16b32556cf8d7257b17cf2bdb7ab7976af4af41ebe79f9796c218f7e91"}, + {file = "orjson-3.10.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:479fd0844ddc3ca77e0fd99644c7fe2de8e8be1efcd57705b5c92e5186e8a250"}, + {file = "orjson-3.10.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fdf5197a21dd660cf19dfd2a3ce79574588f8f5e2dbf21bda9ee2d2b46924d84"}, + {file = "orjson-3.10.7-cp310-none-win32.whl", hash = "sha256:d374d36726746c81a49f3ff8daa2898dccab6596864ebe43d50733275c629175"}, + {file = "orjson-3.10.7-cp310-none-win_amd64.whl", hash = "sha256:cb61938aec8b0ffb6eef484d480188a1777e67b05d58e41b435c74b9d84e0b9c"}, + {file = "orjson-3.10.7-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:7db8539039698ddfb9a524b4dd19508256107568cdad24f3682d5773e60504a2"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:480f455222cb7a1dea35c57a67578848537d2602b46c464472c995297117fa09"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8a9c9b168b3a19e37fe2778c0003359f07822c90fdff8f98d9d2a91b3144d8e0"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8de062de550f63185e4c1c54151bdddfc5625e37daf0aa1e75d2a1293e3b7d9a"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b0dd04483499d1de9c8f6203f8975caf17a6000b9c0c54630cef02e44ee624e"}, + {file = "orjson-3.10.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b58d3795dafa334fc8fd46f7c5dc013e6ad06fd5b9a4cc98cb1456e7d3558bd6"}, + {file = "orjson-3.10.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:33cfb96c24034a878d83d1a9415799a73dc77480e6c40417e5dda0710d559ee6"}, + {file = "orjson-3.10.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e724cebe1fadc2b23c6f7415bad5ee6239e00a69f30ee423f319c6af70e2a5c0"}, + {file = "orjson-3.10.7-cp311-none-win32.whl", hash = "sha256:82763b46053727a7168d29c772ed5c870fdae2f61aa8a25994c7984a19b1021f"}, + {file = "orjson-3.10.7-cp311-none-win_amd64.whl", hash = "sha256:eb8d384a24778abf29afb8e41d68fdd9a156cf6e5390c04cc07bbc24b89e98b5"}, + {file = "orjson-3.10.7-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:44a96f2d4c3af51bfac6bc4ef7b182aa33f2f054fd7f34cc0ee9a320d051d41f"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76ac14cd57df0572453543f8f2575e2d01ae9e790c21f57627803f5e79b0d3c3"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bdbb61dcc365dd9be94e8f7df91975edc9364d6a78c8f7adb69c1cdff318ec93"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b48b3db6bb6e0a08fa8c83b47bc169623f801e5cc4f24442ab2b6617da3b5313"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23820a1563a1d386414fef15c249040042b8e5d07b40ab3fe3efbfbbcbcb8864"}, + {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0c6a008e91d10a2564edbb6ee5069a9e66df3fbe11c9a005cb411f441fd2c09"}, + {file = "orjson-3.10.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d352ee8ac1926d6193f602cbe36b1643bbd1bbcb25e3c1a657a4390f3000c9a5"}, + {file = "orjson-3.10.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d2d9f990623f15c0ae7ac608103c33dfe1486d2ed974ac3f40b693bad1a22a7b"}, + {file = "orjson-3.10.7-cp312-none-win32.whl", hash = "sha256:7c4c17f8157bd520cdb7195f75ddbd31671997cbe10aee559c2d613592e7d7eb"}, + {file = "orjson-3.10.7-cp312-none-win_amd64.whl", hash = "sha256:1d9c0e733e02ada3ed6098a10a8ee0052dd55774de3d9110d29868d24b17faa1"}, + {file = "orjson-3.10.7-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:77d325ed866876c0fa6492598ec01fe30e803272a6e8b10e992288b009cbe149"}, + {file = "orjson-3.10.7-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ea2c232deedcb605e853ae1db2cc94f7390ac776743b699b50b071b02bea6fe"}, + {file = "orjson-3.10.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3dcfbede6737fdbef3ce9c37af3fb6142e8e1ebc10336daa05872bfb1d87839c"}, + {file = "orjson-3.10.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:11748c135f281203f4ee695b7f80bb1358a82a63905f9f0b794769483ea854ad"}, + {file = "orjson-3.10.7-cp313-none-win32.whl", hash = "sha256:a7e19150d215c7a13f39eb787d84db274298d3f83d85463e61d277bbd7f401d2"}, + {file = "orjson-3.10.7-cp313-none-win_amd64.whl", hash = "sha256:eef44224729e9525d5261cc8d28d6b11cafc90e6bd0be2157bde69a52ec83024"}, + {file = "orjson-3.10.7-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6ea2b2258eff652c82652d5e0f02bd5e0463a6a52abb78e49ac288827aaa1469"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:430ee4d85841e1483d487e7b81401785a5dfd69db5de01314538f31f8fbf7ee1"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4b6146e439af4c2472c56f8540d799a67a81226e11992008cb47e1267a9b3225"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:084e537806b458911137f76097e53ce7bf5806dda33ddf6aaa66a028f8d43a23"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4829cf2195838e3f93b70fd3b4292156fc5e097aac3739859ac0dcc722b27ac0"}, + {file = "orjson-3.10.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1193b2416cbad1a769f868b1749535d5da47626ac29445803dae7cc64b3f5c98"}, + {file = "orjson-3.10.7-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:4e6c3da13e5a57e4b3dca2de059f243ebec705857522f188f0180ae88badd354"}, + {file = "orjson-3.10.7-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c31008598424dfbe52ce8c5b47e0752dca918a4fdc4a2a32004efd9fab41d866"}, + {file = "orjson-3.10.7-cp38-none-win32.whl", hash = "sha256:7122a99831f9e7fe977dc45784d3b2edc821c172d545e6420c375e5a935f5a1c"}, + {file = "orjson-3.10.7-cp38-none-win_amd64.whl", hash = "sha256:a763bc0e58504cc803739e7df040685816145a6f3c8a589787084b54ebc9f16e"}, + {file = "orjson-3.10.7-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e76be12658a6fa376fcd331b1ea4e58f5a06fd0220653450f0d415b8fd0fbe20"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed350d6978d28b92939bfeb1a0570c523f6170efc3f0a0ef1f1df287cd4f4960"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:144888c76f8520e39bfa121b31fd637e18d4cc2f115727865fdf9fa325b10412"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:09b2d92fd95ad2402188cf51573acde57eb269eddabaa60f69ea0d733e789fe9"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5b24a579123fa884f3a3caadaed7b75eb5715ee2b17ab5c66ac97d29b18fe57f"}, + {file = "orjson-3.10.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e72591bcfe7512353bd609875ab38050efe3d55e18934e2f18950c108334b4ff"}, + {file = "orjson-3.10.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f4db56635b58cd1a200b0a23744ff44206ee6aa428185e2b6c4a65b3197abdcd"}, + {file = "orjson-3.10.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:0fa5886854673222618638c6df7718ea7fe2f3f2384c452c9ccedc70b4a510a5"}, + {file = "orjson-3.10.7-cp39-none-win32.whl", hash = "sha256:8272527d08450ab16eb405f47e0f4ef0e5ff5981c3d82afe0efd25dcbef2bcd2"}, + {file = "orjson-3.10.7-cp39-none-win_amd64.whl", hash = "sha256:974683d4618c0c7dbf4f69c95a979734bf183d0658611760017f6e70a145af58"}, + {file = "orjson-3.10.7.tar.gz", hash = "sha256:75ef0640403f945f3a1f9f6400686560dbfb0fb5b16589ad62cd477043c4eee3"}, +] + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "psycopg" +version = "3.2.1" +description = "PostgreSQL database adapter for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "psycopg-3.2.1-py3-none-any.whl", hash = "sha256:ece385fb413a37db332f97c49208b36cf030ff02b199d7635ed2fbd378724175"}, + {file = "psycopg-3.2.1.tar.gz", hash = "sha256:dc8da6dc8729dacacda3cc2f17d2c9397a70a66cf0d2b69c91065d60d5f00cb7"}, +] + +[package.dependencies] +typing-extensions = ">=4.4" +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.2.1)"] +c = ["psycopg-c (==3.2.1)"] +dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=1.6)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=4.0)", "mypy (>=1.6)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + +[[package]] +name = "psycopg-pool" +version = "3.2.2" +description = "Connection Pool for Psycopg" +optional = false +python-versions = ">=3.8" +files = [ + {file = "psycopg_pool-3.2.2-py3-none-any.whl", hash = "sha256:273081d0fbfaced4f35e69200c89cb8fbddfe277c38cc86c235b90a2ec2c8153"}, + {file = "psycopg_pool-3.2.2.tar.gz", hash = "sha256:9e22c370045f6d7f2666a5ad1b0caf345f9f1912195b0b25d0d3bcc4f3a7389c"}, +] + +[package.dependencies] +typing-extensions = ">=4.4" + +[[package]] +name = "pydantic" +version = "2.9.0" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.9.0-py3-none-any.whl", hash = "sha256:f66a7073abd93214a20c5f7b32d56843137a7a2e70d02111f3be287035c45370"}, + {file = "pydantic-2.9.0.tar.gz", hash = "sha256:c7a8a9fdf7d100afa49647eae340e2d23efa382466a8d177efcd1381e9be5598"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.23.2" +typing-extensions = [ + {version = ">=4.6.1", markers = "python_version < \"3.13\""}, + {version = ">=4.12.2", markers = "python_version >= \"3.13\""}, +] +tzdata = {version = "*", markers = "python_version >= \"3.9\""} + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.23.2" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.23.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7d0324a35ab436c9d768753cbc3c47a865a2cbc0757066cb864747baa61f6ece"}, + {file = "pydantic_core-2.23.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:276ae78153a94b664e700ac362587c73b84399bd1145e135287513442e7dfbc7"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:964c7aa318da542cdcc60d4a648377ffe1a2ef0eb1e996026c7f74507b720a78"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1cf842265a3a820ebc6388b963ead065f5ce8f2068ac4e1c713ef77a67b71f7c"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae90b9e50fe1bd115b24785e962b51130340408156d34d67b5f8f3fa6540938e"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ae65fdfb8a841556b52935dfd4c3f79132dc5253b12c0061b96415208f4d622"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c8aa40f6ca803f95b1c1c5aeaee6237b9e879e4dfb46ad713229a63651a95fb"}, + {file = "pydantic_core-2.23.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c53100c8ee5a1e102766abde2158077d8c374bee0639201f11d3032e3555dfbc"}, + {file = "pydantic_core-2.23.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6b9dd6aa03c812017411734e496c44fef29b43dba1e3dd1fa7361bbacfc1354"}, + {file = "pydantic_core-2.23.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b18cf68255a476b927910c6873d9ed00da692bb293c5b10b282bd48a0afe3ae2"}, + {file = "pydantic_core-2.23.2-cp310-none-win32.whl", hash = "sha256:e460475719721d59cd54a350c1f71c797c763212c836bf48585478c5514d2854"}, + {file = "pydantic_core-2.23.2-cp310-none-win_amd64.whl", hash = "sha256:5f3cf3721eaf8741cffaf092487f1ca80831202ce91672776b02b875580e174a"}, + {file = "pydantic_core-2.23.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7ce8e26b86a91e305858e018afc7a6e932f17428b1eaa60154bd1f7ee888b5f8"}, + {file = "pydantic_core-2.23.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7e9b24cca4037a561422bf5dc52b38d390fb61f7bfff64053ce1b72f6938e6b2"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753294d42fb072aa1775bfe1a2ba1012427376718fa4c72de52005a3d2a22178"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:257d6a410a0d8aeb50b4283dea39bb79b14303e0fab0f2b9d617701331ed1515"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c8319e0bd6a7b45ad76166cc3d5d6a36c97d0c82a196f478c3ee5346566eebfd"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a05c0240f6c711eb381ac392de987ee974fa9336071fb697768dfdb151345ce"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d5b0ff3218858859910295df6953d7bafac3a48d5cd18f4e3ed9999efd2245f"}, + {file = "pydantic_core-2.23.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:96ef39add33ff58cd4c112cbac076726b96b98bb8f1e7f7595288dcfb2f10b57"}, + {file = "pydantic_core-2.23.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0102e49ac7d2df3379ef8d658d3bc59d3d769b0bdb17da189b75efa861fc07b4"}, + {file = "pydantic_core-2.23.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a6612c2a844043e4d10a8324c54cdff0042c558eef30bd705770793d70b224aa"}, + {file = "pydantic_core-2.23.2-cp311-none-win32.whl", hash = "sha256:caffda619099cfd4f63d48462f6aadbecee3ad9603b4b88b60cb821c1b258576"}, + {file = "pydantic_core-2.23.2-cp311-none-win_amd64.whl", hash = "sha256:6f80fba4af0cb1d2344869d56430e304a51396b70d46b91a55ed4959993c0589"}, + {file = "pydantic_core-2.23.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:4c83c64d05ffbbe12d4e8498ab72bdb05bcc1026340a4a597dc647a13c1605ec"}, + {file = "pydantic_core-2.23.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6294907eaaccf71c076abdd1c7954e272efa39bb043161b4b8aa1cd76a16ce43"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a801c5e1e13272e0909c520708122496647d1279d252c9e6e07dac216accc41"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cc0c316fba3ce72ac3ab7902a888b9dc4979162d320823679da270c2d9ad0cad"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b06c5d4e8701ac2ba99a2ef835e4e1b187d41095a9c619c5b185c9068ed2a49"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82764c0bd697159fe9947ad59b6db6d7329e88505c8f98990eb07e84cc0a5d81"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b1a195efd347ede8bcf723e932300292eb13a9d2a3c1f84eb8f37cbbc905b7f"}, + {file = "pydantic_core-2.23.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b7efb12e5071ad8d5b547487bdad489fbd4a5a35a0fc36a1941517a6ad7f23e0"}, + {file = "pydantic_core-2.23.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5dd0ec5f514ed40e49bf961d49cf1bc2c72e9b50f29a163b2cc9030c6742aa73"}, + {file = "pydantic_core-2.23.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:820f6ee5c06bc868335e3b6e42d7ef41f50dfb3ea32fbd523ab679d10d8741c0"}, + {file = "pydantic_core-2.23.2-cp312-none-win32.whl", hash = "sha256:3713dc093d5048bfaedbba7a8dbc53e74c44a140d45ede020dc347dda18daf3f"}, + {file = "pydantic_core-2.23.2-cp312-none-win_amd64.whl", hash = "sha256:e1895e949f8849bc2757c0dbac28422a04be031204df46a56ab34bcf98507342"}, + {file = "pydantic_core-2.23.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:da43cbe593e3c87d07108d0ebd73771dc414488f1f91ed2e204b0370b94b37ac"}, + {file = "pydantic_core-2.23.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:64d094ea1aa97c6ded4748d40886076a931a8bf6f61b6e43e4a1041769c39dd2"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:084414ffe9a85a52940b49631321d636dadf3576c30259607b75516d131fecd0"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:043ef8469f72609c4c3a5e06a07a1f713d53df4d53112c6d49207c0bd3c3bd9b"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3649bd3ae6a8ebea7dc381afb7f3c6db237fc7cebd05c8ac36ca8a4187b03b30"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6db09153d8438425e98cdc9a289c5fade04a5d2128faff8f227c459da21b9703"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5668b3173bb0b2e65020b60d83f5910a7224027232c9f5dc05a71a1deac9f960"}, + {file = "pydantic_core-2.23.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1c7b81beaf7c7ebde978377dc53679c6cba0e946426fc7ade54251dfe24a7604"}, + {file = "pydantic_core-2.23.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:ae579143826c6f05a361d9546446c432a165ecf1c0b720bbfd81152645cb897d"}, + {file = "pydantic_core-2.23.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:19f1352fe4b248cae22a89268720fc74e83f008057a652894f08fa931e77dced"}, + {file = "pydantic_core-2.23.2-cp313-none-win32.whl", hash = "sha256:e1a79ad49f346aa1a2921f31e8dbbab4d64484823e813a002679eaa46cba39e1"}, + {file = "pydantic_core-2.23.2-cp313-none-win_amd64.whl", hash = "sha256:582871902e1902b3c8e9b2c347f32a792a07094110c1bca6c2ea89b90150caac"}, + {file = "pydantic_core-2.23.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:743e5811b0c377eb830150d675b0847a74a44d4ad5ab8845923d5b3a756d8100"}, + {file = "pydantic_core-2.23.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6650a7bbe17a2717167e3e23c186849bae5cef35d38949549f1c116031b2b3aa"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56e6a12ec8d7679f41b3750ffa426d22b44ef97be226a9bab00a03365f217b2b"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:810ca06cca91de9107718dc83d9ac4d2e86efd6c02cba49a190abcaf33fb0472"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:785e7f517ebb9890813d31cb5d328fa5eda825bb205065cde760b3150e4de1f7"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ef71ec876fcc4d3bbf2ae81961959e8d62f8d74a83d116668409c224012e3af"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d50ac34835c6a4a0d456b5db559b82047403c4317b3bc73b3455fefdbdc54b0a"}, + {file = "pydantic_core-2.23.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16b25a4a120a2bb7dab51b81e3d9f3cde4f9a4456566c403ed29ac81bf49744f"}, + {file = "pydantic_core-2.23.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:41ae8537ad371ec018e3c5da0eb3f3e40ee1011eb9be1da7f965357c4623c501"}, + {file = "pydantic_core-2.23.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07049ec9306ec64e955b2e7c40c8d77dd78ea89adb97a2013d0b6e055c5ee4c5"}, + {file = "pydantic_core-2.23.2-cp38-none-win32.whl", hash = "sha256:086c5db95157dc84c63ff9d96ebb8856f47ce113c86b61065a066f8efbe80acf"}, + {file = "pydantic_core-2.23.2-cp38-none-win_amd64.whl", hash = "sha256:67b6655311b00581914aba481729971b88bb8bc7996206590700a3ac85e457b8"}, + {file = "pydantic_core-2.23.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:358331e21a897151e54d58e08d0219acf98ebb14c567267a87e971f3d2a3be59"}, + {file = "pydantic_core-2.23.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c4d9f15ffe68bcd3898b0ad7233af01b15c57d91cd1667f8d868e0eacbfe3f87"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0123655fedacf035ab10c23450163c2f65a4174f2bb034b188240a6cf06bb123"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e6e3ccebdbd6e53474b0bb7ab8b88e83c0cfe91484b25e058e581348ee5a01a5"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc535cb898ef88333cf317777ecdfe0faac1c2a3187ef7eb061b6f7ecf7e6bae"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aab9e522efff3993a9e98ab14263d4e20211e62da088298089a03056980a3e69"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05b366fb8fe3d8683b11ac35fa08947d7b92be78ec64e3277d03bd7f9b7cda79"}, + {file = "pydantic_core-2.23.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7568f682c06f10f30ef643a1e8eec4afeecdafde5c4af1b574c6df079e96f96c"}, + {file = "pydantic_core-2.23.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:cdd02a08205dc90238669f082747612cb3c82bd2c717adc60f9b9ecadb540f80"}, + {file = "pydantic_core-2.23.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1a2ab4f410f4b886de53b6bddf5dd6f337915a29dd9f22f20f3099659536b2f6"}, + {file = "pydantic_core-2.23.2-cp39-none-win32.whl", hash = "sha256:0448b81c3dfcde439551bb04a9f41d7627f676b12701865c8a2574bcea034437"}, + {file = "pydantic_core-2.23.2-cp39-none-win_amd64.whl", hash = "sha256:4cebb9794f67266d65e7e4cbe5dcf063e29fc7b81c79dc9475bd476d9534150e"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e758d271ed0286d146cf7c04c539a5169a888dd0b57026be621547e756af55bc"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f477d26183e94eaafc60b983ab25af2a809a1b48ce4debb57b343f671b7a90b6"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da3131ef2b940b99106f29dfbc30d9505643f766704e14c5d5e504e6a480c35e"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:329a721253c7e4cbd7aad4a377745fbcc0607f9d72a3cc2102dd40519be75ed2"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7706e15cdbf42f8fab1e6425247dfa98f4a6f8c63746c995d6a2017f78e619ae"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e64ffaf8f6e17ca15eb48344d86a7a741454526f3a3fa56bc493ad9d7ec63936"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:dd59638025160056687d598b054b64a79183f8065eae0d3f5ca523cde9943940"}, + {file = "pydantic_core-2.23.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:12625e69b1199e94b0ae1c9a95d000484ce9f0182f9965a26572f054b1537e44"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5d813fd871b3d5c3005157622ee102e8908ad6011ec915a18bd8fde673c4360e"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:1eb37f7d6a8001c0f86dc8ff2ee8d08291a536d76e49e78cda8587bb54d8b329"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ce7eaf9a98680b4312b7cebcdd9352531c43db00fca586115845df388f3c465"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f087879f1ffde024dd2788a30d55acd67959dcf6c431e9d3682d1c491a0eb474"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6ce883906810b4c3bd90e0ada1f9e808d9ecf1c5f0b60c6b8831d6100bcc7dd6"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:a8031074a397a5925d06b590121f8339d34a5a74cfe6970f8a1124eb8b83f4ac"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:23af245b8f2f4ee9e2c99cb3f93d0e22fb5c16df3f2f643f5a8da5caff12a653"}, + {file = "pydantic_core-2.23.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c57e493a0faea1e4c38f860d6862ba6832723396c884fbf938ff5e9b224200e2"}, + {file = "pydantic_core-2.23.2.tar.gz", hash = "sha256:95d6bf449a1ac81de562d65d180af5d8c19672793c81877a2eda8fde5d08f2fd"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pytest" +version = "7.4.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "pytest-watcher" +version = "0.4.2" +description = "Automatically rerun your tests on file modifications" +optional = false +python-versions = "<4.0.0,>=3.7.0" +files = [ + {file = "pytest_watcher-0.4.2-py3-none-any.whl", hash = "sha256:a43949ba67dd8d7e1fd0de5eea44a999081f0aec9f93b4e744264b4c6a3d9bbe"}, + {file = "pytest_watcher-0.4.2.tar.gz", hash = "sha256:7b292f025ca19617cd7567c228c6187b5087f2da9e4d2cf6e144e5764a0471b0"}, +] + +[package.dependencies] +tomli = {version = ">=2.0.1,<3.0.0", markers = "python_version < \"3.11\""} +watchdog = ">=2.0.0" + +[[package]] +name = "pyyaml" +version = "6.0.2" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "ruff" +version = "0.6.2" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.6.2-py3-none-linux_armv6l.whl", hash = "sha256:5c8cbc6252deb3ea840ad6a20b0f8583caab0c5ef4f9cca21adc5a92b8f79f3c"}, + {file = "ruff-0.6.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:17002fe241e76544448a8e1e6118abecbe8cd10cf68fde635dad480dba594570"}, + {file = "ruff-0.6.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3dbeac76ed13456f8158b8f4fe087bf87882e645c8e8b606dd17b0b66c2c1158"}, + {file = "ruff-0.6.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:094600ee88cda325988d3f54e3588c46de5c18dae09d683ace278b11f9d4d534"}, + {file = "ruff-0.6.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:316d418fe258c036ba05fbf7dfc1f7d3d4096db63431546163b472285668132b"}, + {file = "ruff-0.6.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d72b8b3abf8a2d51b7b9944a41307d2f442558ccb3859bbd87e6ae9be1694a5d"}, + {file = "ruff-0.6.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2aed7e243be68487aa8982e91c6e260982d00da3f38955873aecd5a9204b1d66"}, + {file = "ruff-0.6.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d371f7fc9cec83497fe7cf5eaf5b76e22a8efce463de5f775a1826197feb9df8"}, + {file = "ruff-0.6.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8f310d63af08f583363dfb844ba8f9417b558199c58a5999215082036d795a1"}, + {file = "ruff-0.6.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7db6880c53c56addb8638fe444818183385ec85eeada1d48fc5abe045301b2f1"}, + {file = "ruff-0.6.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1175d39faadd9a50718f478d23bfc1d4da5743f1ab56af81a2b6caf0a2394f23"}, + {file = "ruff-0.6.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:5b939f9c86d51635fe486585389f54582f0d65b8238e08c327c1534844b3bb9a"}, + {file = "ruff-0.6.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d0d62ca91219f906caf9b187dea50d17353f15ec9bb15aae4a606cd697b49b4c"}, + {file = "ruff-0.6.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7438a7288f9d67ed3c8ce4d059e67f7ed65e9fe3aa2ab6f5b4b3610e57e3cb56"}, + {file = "ruff-0.6.2-py3-none-win32.whl", hash = "sha256:279d5f7d86696df5f9549b56b9b6a7f6c72961b619022b5b7999b15db392a4da"}, + {file = "ruff-0.6.2-py3-none-win_amd64.whl", hash = "sha256:d9f3469c7dd43cd22eb1c3fc16926fb8258d50cb1b216658a07be95dd117b0f2"}, + {file = "ruff-0.6.2-py3-none-win_arm64.whl", hash = "sha256:f28fcd2cd0e02bdf739297516d5643a945cc7caf09bd9bcb4d932540a5ea4fa9"}, + {file = "ruff-0.6.2.tar.gz", hash = "sha256:239ee6beb9e91feb8e0ec384204a763f36cb53fb895a1a364618c6abb076b3be"}, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + +[[package]] +name = "tenacity" +version = "8.5.0" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687"}, + {file = "tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "watchdog" +version = "4.0.1" +description = "Filesystem events monitoring" +optional = false +python-versions = ">=3.8" +files = [ + {file = "watchdog-4.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:da2dfdaa8006eb6a71051795856bedd97e5b03e57da96f98e375682c48850645"}, + {file = "watchdog-4.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e93f451f2dfa433d97765ca2634628b789b49ba8b504fdde5837cdcf25fdb53b"}, + {file = "watchdog-4.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ef0107bbb6a55f5be727cfc2ef945d5676b97bffb8425650dadbb184be9f9a2b"}, + {file = "watchdog-4.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:17e32f147d8bf9657e0922c0940bcde863b894cd871dbb694beb6704cfbd2fb5"}, + {file = "watchdog-4.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03e70d2df2258fb6cb0e95bbdbe06c16e608af94a3ffbd2b90c3f1e83eb10767"}, + {file = "watchdog-4.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:123587af84260c991dc5f62a6e7ef3d1c57dfddc99faacee508c71d287248459"}, + {file = "watchdog-4.0.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:093b23e6906a8b97051191a4a0c73a77ecc958121d42346274c6af6520dec175"}, + {file = "watchdog-4.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:611be3904f9843f0529c35a3ff3fd617449463cb4b73b1633950b3d97fa4bfb7"}, + {file = "watchdog-4.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:62c613ad689ddcb11707f030e722fa929f322ef7e4f18f5335d2b73c61a85c28"}, + {file = "watchdog-4.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d4925e4bf7b9bddd1c3de13c9b8a2cdb89a468f640e66fbfabaf735bd85b3e35"}, + {file = "watchdog-4.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cad0bbd66cd59fc474b4a4376bc5ac3fc698723510cbb64091c2a793b18654db"}, + {file = "watchdog-4.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a3c2c317a8fb53e5b3d25790553796105501a235343f5d2bf23bb8649c2c8709"}, + {file = "watchdog-4.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c9904904b6564d4ee8a1ed820db76185a3c96e05560c776c79a6ce5ab71888ba"}, + {file = "watchdog-4.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:667f3c579e813fcbad1b784db7a1aaa96524bed53437e119f6a2f5de4db04235"}, + {file = "watchdog-4.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d10a681c9a1d5a77e75c48a3b8e1a9f2ae2928eda463e8d33660437705659682"}, + {file = "watchdog-4.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0144c0ea9997b92615af1d94afc0c217e07ce2c14912c7b1a5731776329fcfc7"}, + {file = "watchdog-4.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:998d2be6976a0ee3a81fb8e2777900c28641fb5bfbd0c84717d89bca0addcdc5"}, + {file = "watchdog-4.0.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e7921319fe4430b11278d924ef66d4daa469fafb1da679a2e48c935fa27af193"}, + {file = "watchdog-4.0.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:f0de0f284248ab40188f23380b03b59126d1479cd59940f2a34f8852db710625"}, + {file = "watchdog-4.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bca36be5707e81b9e6ce3208d92d95540d4ca244c006b61511753583c81c70dd"}, + {file = "watchdog-4.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ab998f567ebdf6b1da7dc1e5accfaa7c6992244629c0fdaef062f43249bd8dee"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:dddba7ca1c807045323b6af4ff80f5ddc4d654c8bce8317dde1bd96b128ed253"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_armv7l.whl", hash = "sha256:4513ec234c68b14d4161440e07f995f231be21a09329051e67a2118a7a612d2d"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_i686.whl", hash = "sha256:4107ac5ab936a63952dea2a46a734a23230aa2f6f9db1291bf171dac3ebd53c6"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_ppc64.whl", hash = "sha256:6e8c70d2cd745daec2a08734d9f63092b793ad97612470a0ee4cbb8f5f705c57"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:f27279d060e2ab24c0aa98363ff906d2386aa6c4dc2f1a374655d4e02a6c5e5e"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_s390x.whl", hash = "sha256:f8affdf3c0f0466e69f5b3917cdd042f89c8c63aebdb9f7c078996f607cdb0f5"}, + {file = "watchdog-4.0.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ac7041b385f04c047fcc2951dc001671dee1b7e0615cde772e84b01fbf68ee84"}, + {file = "watchdog-4.0.1-py3-none-win32.whl", hash = "sha256:206afc3d964f9a233e6ad34618ec60b9837d0582b500b63687e34011e15bb429"}, + {file = "watchdog-4.0.1-py3-none-win_amd64.whl", hash = "sha256:7577b3c43e5909623149f76b099ac49a1a01ca4e167d1785c76eb52fa585745a"}, + {file = "watchdog-4.0.1-py3-none-win_ia64.whl", hash = "sha256:d7b9f5f3299e8dd230880b6c55504a1f69cf1e4316275d1b215ebdd8187ec88d"}, + {file = "watchdog-4.0.1.tar.gz", hash = "sha256:eebaacf674fa25511e8867028d281e602ee6500045b57f43b08778082f7f8b44"}, +] + +[package.extras] +watchmedo = ["PyYAML (>=3.10)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9.0,<4.0" +content-hash = "c6f14b1de3fa0c90d7b8d860b85003b5b1261c0f64ccdf489777e213419bf278" diff --git a/libs/scheduler-kafka/pyproject.toml b/libs/scheduler-kafka/pyproject.toml new file mode 100644 index 000000000..d107afcc8 --- /dev/null +++ b/libs/scheduler-kafka/pyproject.toml @@ -0,0 +1,57 @@ +[tool.poetry] +name = "langgraph-scheduler-kafka" +version = "1.0.0" +description = "Library with Kafka-based work scheduler." +authors = [] +license = "MIT" +readme = "README.md" +repository = "https://www.github.com/langchain-ai/langgraph" +packages = [{ include = "langgraph" }] + +[tool.poetry.dependencies] +python = "^3.9.0,<4.0" +orjson = "^3.10.7" +crc32c = "^2.7.post1" +aiokafka = "^0.11.0" +kafka-python-ng = "^2.2.2" + +[tool.poetry.group.dev.dependencies] +ruff = "^0.6.2" +codespell = "^2.2.0" +pytest = "^7.2.1" +pytest-mock = "^3.11.1" +pytest-watcher = "^0.4.1" +mypy = "^1.10.0" +langgraph = {path = "../langgraph", develop = true} +langgraph-checkpoint-postgres = {path = "../checkpoint-postgres", develop = true} +langgraph-checkpoint = {path = "../checkpoint", develop = true} + +[tool.pytest.ini_options] +# --strict-markers will raise errors on unknown marks. +# https://docs.pytest.org/en/7.1.x/how-to/mark.html#raising-errors-on-unknown-marks +# +# https://docs.pytest.org/en/7.1.x/reference/reference.html +# --strict-config any warnings encountered while parsing the `pytest` +# section of the configuration file raise errors. +addopts = "--strict-markers --strict-config --durations=5 -vv" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.ruff] +lint.select = [ + "E", # pycodestyle + "F", # Pyflakes + "UP", # pyupgrade + "B", # flake8-bugbear + "I", # isort +] +lint.ignore = ["E501", "B008", "UP007", "UP006"] + +[tool.pytest-watcher] +now = true +delay = 0.1 +runner_args = ["-s", "--ff", "-v", "--tb", "short"] +patterns = ["*.py"] diff --git a/libs/scheduler-kafka/testss/__init__.py b/libs/scheduler-kafka/testss/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/libs/scheduler-kafka/testss/compose.yml b/libs/scheduler-kafka/testss/compose.yml new file mode 100644 index 000000000..59571371e --- /dev/null +++ b/libs/scheduler-kafka/testss/compose.yml @@ -0,0 +1,20 @@ +services: + broker: + image: apache/kafka:latest + ports: + - "9092:9092" + postgres: + image: postgres:16 + ports: + - "5443:5432" + environment: + POSTGRES_DB: postgres + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + healthcheck: + test: pg_isready -U postgres + start_period: 10s + timeout: 1s + retries: 5 + interval: 60s + start_interval: 1s diff --git a/libs/scheduler-kafka/testss/conftest.py b/libs/scheduler-kafka/testss/conftest.py new file mode 100644 index 000000000..4296b9851 --- /dev/null +++ b/libs/scheduler-kafka/testss/conftest.py @@ -0,0 +1,58 @@ +from typing import AsyncIterator, Iterator +from uuid import uuid4 + +import kafka.admin +import pytest +from psycopg import AsyncConnection + +from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver +from langgraph.scheduler.kafka.types import Topics + +DEFAULT_POSTGRES_URI = "postgres://postgres:postgres@localhost:5443/" + + +@pytest.fixture +def anyio_backend(): + return "asyncio" + + +@pytest.fixture +def topics() -> Iterator[Topics]: + o = f"test_{uuid4().hex[:16]}" + e = f"test_{uuid4().hex[:16]}" + admin = kafka.admin.KafkaAdminClient() + # create topics + admin.create_topics( + [ + kafka.admin.NewTopic(name=o, num_partitions=1, replication_factor=1), + kafka.admin.NewTopic(name=e, num_partitions=1, replication_factor=1), + ] + ) + # yield topics + yield Topics(orchestrator=o, executor=e) + # delete topics + admin.delete_topics([o, e]) + admin.close() + + +@pytest.fixture +async def checkpointer() -> AsyncIterator[AsyncPostgresSaver]: + database = f"test_{uuid4().hex[:16]}" + # create unique db + async with await AsyncConnection.connect( + DEFAULT_POSTGRES_URI, autocommit=True + ) as conn: + await conn.execute(f"CREATE DATABASE {database}") + try: + # yield checkpointer + async with AsyncPostgresSaver.from_conn_string( + DEFAULT_POSTGRES_URI + database + ) as checkpointer: + await checkpointer.setup() + yield checkpointer + finally: + # drop unique db + async with await AsyncConnection.connect( + DEFAULT_POSTGRES_URI, autocommit=True + ) as conn: + await conn.execute(f"DROP DATABASE {database}") diff --git a/libs/scheduler-kafka/testss/test_scheduler.py b/libs/scheduler-kafka/testss/test_scheduler.py new file mode 100644 index 000000000..b53abd361 --- /dev/null +++ b/libs/scheduler-kafka/testss/test_scheduler.py @@ -0,0 +1,124 @@ +import asyncio +import operator +from typing import Annotated, TypedDict, Union + +import pytest +from aiokafka import AIOKafkaProducer + +from langgraph.checkpoint.base import BaseCheckpointSaver +from langgraph.graph.state import StateGraph +from langgraph.pregel import Pregel +from langgraph.scheduler.kafka import serde +from langgraph.scheduler.kafka.executor import KafkaExecutor +from langgraph.scheduler.kafka.orchestrator import KafkaOrchestrator +from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics + +pytestmark = pytest.mark.anyio + + +def mk_fanout_graph(checkpointer: BaseCheckpointSaver) -> Pregel: + # copied from test_in_one_fan_out_state_graph_waiting_edge_multiple_cond_edge + def sorted_add( + x: list[str], y: Union[list[str], list[tuple[str, str]]] + ) -> list[str]: + if isinstance(y[0], tuple): + for rem, _ in y: + x.remove(rem) + y = [t[1] for t in y] + return sorted(operator.add(x, y)) + + class State(TypedDict, total=False): + query: str + answer: str + docs: Annotated[list[str], sorted_add] + + async def rewrite_query(data: State) -> State: + print("rewrite_query", data) + return {"query": f'query: {data["query"]}'} + + async def retriever_picker(data: State) -> list[str]: + return ["analyzer_one", "retriever_two"] + + async def analyzer_one(data: State) -> State: + return {"query": f'analyzed: {data["query"]}'} + + async def retriever_one(data: State) -> State: + return {"docs": ["doc1", "doc2"]} + + async def retriever_two(data: State) -> State: + await asyncio.sleep(0.1) + return {"docs": ["doc3", "doc4"]} + + async def qa(data: State) -> State: + return {"answer": ",".join(data["docs"])} + + async def decider(data: State) -> None: + return None + + def decider_cond(data: State) -> str: + if data["query"].count("analyzed") > 1: + return "qa" + else: + return "rewrite_query" + + builder = StateGraph(State) + + builder.add_node("rewrite_query", rewrite_query) + builder.add_node("analyzer_one", analyzer_one) + builder.add_node("retriever_one", retriever_one) + builder.add_node("retriever_two", retriever_two) + builder.add_node("decider", decider) + builder.add_node("qa", qa) + + builder.set_entry_point("rewrite_query") + builder.add_conditional_edges("rewrite_query", retriever_picker) + builder.add_edge("analyzer_one", "retriever_one") + builder.add_edge(["retriever_one", "retriever_two"], "decider") + builder.add_conditional_edges("decider", decider_cond) + builder.set_finish_point("qa") + + return builder.compile(checkpointer) + + +async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) -> None: + graph = mk_fanout_graph(checkpointer) + n_orch_msgs = 0 + n_exec_msgs = 0 + + async def orchestrator() -> None: + nonlocal n_orch_msgs + async with KafkaOrchestrator(graph, topics) as orch: + async for msg in orch: + n_orch_msgs += 1 + print("orch", msg) + + async def executor() -> None: + nonlocal n_exec_msgs + async with KafkaExecutor(graph, topics) as exec: + async for msg in exec: + n_exec_msgs += 1 + print("exec", msg) + + async with asyncio.TaskGroup() as tg: + o = tg.create_task(orchestrator(), name="orchestrator") + e = tg.create_task(executor(), name="executor") + + # start a new run + producer = AIOKafkaProducer(value_serializer=serde.dumps) + await producer.start() + await producer.send_and_wait( + topics.orchestrator, + MessageToOrchestrator( + input={"query": "what is weather in sf"}, + config={"configurable": {"thread_id": "1"}}, + ), + ) + await producer.stop() + + # wait for the run to finish + await asyncio.sleep(5) + o.cancel() + e.cancel() + + assert n_orch_msgs == 13 + assert n_exec_msgs == 12 From 381ea9cec8dfbc7f47390ebf0c2bdd5269dd4f64 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Thu, 5 Sep 2024 18:04:23 -0700 Subject: [PATCH 02/34] Rename --- libs/scheduler-kafka/{testss => tests}/__init__.py | 0 libs/scheduler-kafka/{testss => tests}/compose.yml | 0 libs/scheduler-kafka/{testss => tests}/conftest.py | 0 libs/scheduler-kafka/{testss => tests}/test_scheduler.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename libs/scheduler-kafka/{testss => tests}/__init__.py (100%) rename libs/scheduler-kafka/{testss => tests}/compose.yml (100%) rename libs/scheduler-kafka/{testss => tests}/conftest.py (100%) rename libs/scheduler-kafka/{testss => tests}/test_scheduler.py (100%) diff --git a/libs/scheduler-kafka/testss/__init__.py b/libs/scheduler-kafka/tests/__init__.py similarity index 100% rename from libs/scheduler-kafka/testss/__init__.py rename to libs/scheduler-kafka/tests/__init__.py diff --git a/libs/scheduler-kafka/testss/compose.yml b/libs/scheduler-kafka/tests/compose.yml similarity index 100% rename from libs/scheduler-kafka/testss/compose.yml rename to libs/scheduler-kafka/tests/compose.yml diff --git a/libs/scheduler-kafka/testss/conftest.py b/libs/scheduler-kafka/tests/conftest.py similarity index 100% rename from libs/scheduler-kafka/testss/conftest.py rename to libs/scheduler-kafka/tests/conftest.py diff --git a/libs/scheduler-kafka/testss/test_scheduler.py b/libs/scheduler-kafka/tests/test_scheduler.py similarity index 100% rename from libs/scheduler-kafka/testss/test_scheduler.py rename to libs/scheduler-kafka/tests/test_scheduler.py From dc5be3fd58c5d0677ef5acdb20e26c97c2573b8a Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 09:17:05 -0700 Subject: [PATCH 03/34] Rename --- libs/scheduler-kafka/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/scheduler-kafka/Makefile b/libs/scheduler-kafka/Makefile index 53c67e9a4..7462627e1 100644 --- a/libs/scheduler-kafka/Makefile +++ b/libs/scheduler-kafka/Makefile @@ -5,10 +5,10 @@ ###################### start-services: - docker compose -f testss/compose.yml up -V --force-recreate --wait --remove-orphans + docker compose -f tests/compose.yml up -V --force-recreate --wait --remove-orphans stop-services: - docker compose -f testss/compose.yml down + docker compose -f tests/compose.yml down test: make start-services && poetry run pytest; \ From e6d6f565a52f5bf981f317db7f729c330cd9735c Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 12:46:10 -0700 Subject: [PATCH 04/34] Improvements to consumer and producer patterns - await future returned by send() instead of flush() - use consumer groups by default - process tasks in batches by default, configurable - manually commit offsets when batch is processed --- .../langgraph/scheduler/kafka/executor.py | 100 ++++++++++----- .../langgraph/scheduler/kafka/orchestrator.py | 116 +++++++++++------- libs/scheduler-kafka/pyproject.toml | 2 +- libs/scheduler-kafka/tests/test_scheduler.py | 56 +++++---- 4 files changed, 179 insertions(+), 95 deletions(-) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index de1e61e09..82549cad9 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -1,15 +1,17 @@ import asyncio -from contextlib import AbstractAsyncContextManager -from typing import Any +from contextlib import AbstractAsyncContextManager, AsyncExitStack +from functools import partial +from typing import Any, Self, Sequence import aiokafka +from langchain_core.runnables import RunnableConfig import langgraph.scheduler.kafka.serde as serde from langgraph.constants import ERROR from langgraph.errors import TaskNotFound from langgraph.pregel import Pregel from langgraph.pregel.algo import prepare_single_task -from langgraph.pregel.executor import AsyncBackgroundExecutor +from langgraph.pregel.executor import AsyncBackgroundExecutor, Submit from langgraph.pregel.manager import AsyncChannelsManager from langgraph.pregel.runner import PregelRunner from langgraph.scheduler.kafka.types import ( @@ -20,35 +22,68 @@ from langgraph.scheduler.kafka.types import ( class KafkaExecutor(AbstractAsyncContextManager): - def __init__(self, graph: Pregel, topics: Topics, **kwargs: Any) -> None: + def __init__( + self, + graph: Pregel, + topics: Topics, + *, + group_id: str = "executor", + batch_max_n: int = 10, + batch_max_ms: int = 1000, + **kwargs: Any, + ) -> None: self.graph = graph self.topics = topics - self.consumer = aiokafka.AIOKafkaConsumer( - topics.executor, value_deserializer=serde.loads, **kwargs - ) - self.producer = aiokafka.AIOKafkaProducer( - value_serializer=serde.dumps, **kwargs - ) + self.stack = AsyncExitStack() + self.kwargs = kwargs + self.group_id = group_id + self.batch_max_n = batch_max_n + self.batch_max_ms = batch_max_ms - async def __aenter__(self) -> "KafkaExecutor": - await self.consumer.start() - await self.producer.start() + async def __aenter__(self) -> Self: + self.consumer = await self.stack.enter_async_context( + aiokafka.AIOKafkaConsumer( + self.topics.executor, + value_deserializer=serde.loads, + auto_offset_reset="earliest", + group_id=self.group_id, + enable_auto_commit=False, + **self.kwargs, + ) + ) + self.producer = await self.stack.enter_async_context( + aiokafka.AIOKafkaProducer( + value_serializer=serde.dumps, + **self.kwargs, + ) + ) return self async def __aexit__(self, *args: Any) -> None: - await self.consumer.stop() - await self.producer.stop() + await self.stack.__aexit__(*args) - def __aiter__(self) -> "KafkaExecutor": + def __aiter__(self) -> Self: return self - async def __anext__(self) -> Any: - # wait for next message + async def __anext__(self) -> Sequence[MessageToExecutor]: + # wait for next batch try: - rec = await self.consumer.getone() - msg: MessageToExecutor = rec.value + recs = await self.consumer.getmany( + timeout_ms=self.batch_max_ms, max_records=self.batch_max_n + ) + msgs: list[MessageToExecutor] = [ + msg.value for msgs in recs.values() for msg in msgs + ] except aiokafka.ConsumerStoppedError: raise StopAsyncIteration from None + # process batch + await asyncio.gather(*(self.each(msg) for msg in msgs)) + # commit offsets + await self.consumer.commit() + # return message + return msgs + + async def each(self, msg: MessageToExecutor) -> None: # process message saved = await self.graph.checkpointer.aget_tuple(msg["config"]) if saved is None: @@ -56,13 +91,6 @@ class KafkaExecutor(AbstractAsyncContextManager): async with AsyncChannelsManager( self.graph.channels, saved.checkpoint, msg["config"], self.graph.store ) as (channels, managed), AsyncBackgroundExecutor() as submit: - - def put_writes(task_id: str, writes: list[tuple[str, Any]]) -> None: - print("put_writes", task_id, writes) - return submit( - self.graph.checkpointer.aput_writes, msg["config"], writes, task_id - ) - if task := await asyncio.to_thread( prepare_single_task, msg["task"]["path"], @@ -77,7 +105,10 @@ class KafkaExecutor(AbstractAsyncContextManager): is_resuming=msg["task"]["resuming"], ): # execute task, saving writes - runner = PregelRunner(submit=submit, put_writes=put_writes) + runner = PregelRunner( + submit=submit, + put_writes=partial(self._put_writes, submit, msg["config"]), + ) async for _ in runner.atick([task]): pass else: @@ -86,9 +117,16 @@ class KafkaExecutor(AbstractAsyncContextManager): msg["config"], [(ERROR, TaskNotFound())] ) # notify orchestrator - await self.producer.send( + await self.producer.send_and_wait( self.topics.orchestrator, value=MessageToOrchestrator(input=None, config=msg["config"]), ) - # return message - return msg + + def _put_writes( + self, + submit: Submit, + config: RunnableConfig, + task_id: str, + writes: list[tuple[str, Any]], + ) -> None: + return submit(self.graph.checkpointer.aput_writes, config, writes, task_id) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index 5187f0435..b8fe1f003 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -1,5 +1,6 @@ -from contextlib import AbstractAsyncContextManager -from typing import Any +import asyncio +from contextlib import AbstractAsyncContextManager, AsyncExitStack +from typing import Any, Self import aiokafka from langchain_core.runnables import ensure_config @@ -18,35 +19,64 @@ from langgraph.utils.config import patch_configurable class KafkaOrchestrator(AbstractAsyncContextManager): - def __init__(self, graph: Pregel, topics: Topics, **kwargs: Any) -> None: + def __init__( + self, + graph: Pregel, + topics: Topics, + group_id: str = "orchestrator", + batch_max_n: int = 10, + batch_max_ms: int = 1000, + **kwargs: Any, + ) -> None: self.graph = graph self.topics = topics - self.consumer = aiokafka.AIOKafkaConsumer( - topics.orchestrator, value_deserializer=serde.loads, **kwargs - ) - self.producer = aiokafka.AIOKafkaProducer( - value_serializer=serde.dumps, **kwargs - ) + self.stack = AsyncExitStack() + self.kwargs = kwargs + self.group_id = group_id + self.batch_max_n = batch_max_n + self.batch_max_ms = batch_max_ms - async def __aenter__(self) -> "KafkaOrchestrator": - await self.consumer.start() - await self.producer.start() + async def __aenter__(self) -> Self: + self.consumer = await self.stack.enter_async_context( + aiokafka.AIOKafkaConsumer( + self.topics.orchestrator, + value_deserializer=serde.loads, + auto_offset_reset="earliest", + group_id=self.group_id, + enable_auto_commit=False, + **self.kwargs, + ) + ) + self.producer = await self.stack.enter_async_context( + aiokafka.AIOKafkaProducer(value_serializer=serde.dumps, **self.kwargs) + ) return self async def __aexit__(self, *args: Any) -> None: - await self.consumer.stop() - await self.producer.stop() + await self.stack.__aexit__(*args) - def __aiter__(self) -> "KafkaOrchestrator": + def __aiter__(self) -> Self: return self - async def __anext__(self) -> Any: - # wait for next message + async def __anext__(self) -> list[MessageToOrchestrator]: + # wait for next batch try: - rec = await self.consumer.getone() - msg: MessageToOrchestrator = rec.value + recs = await self.consumer.getmany( + timeout_ms=self.batch_max_ms, max_records=self.batch_max_n + ) + msgs: list[MessageToOrchestrator] = [ + msg.value for msgs in recs.values() for msg in msgs + ] except aiokafka.ConsumerStoppedError: raise StopAsyncIteration from None + # process batch + await asyncio.gather(*(self.each(msg) for msg in msgs)) + # commit offsets + await self.consumer.commit() + # return message + return msgs + + async def each(self, msg: MessageToOrchestrator) -> None: # process message async with AsyncPregelLoop( msg["input"], @@ -60,35 +90,37 @@ class KafkaOrchestrator(AbstractAsyncContextManager): stream_keys=self.graph.stream_channels, ) as loop: if loop.tick(input_keys=self.graph.input_channels): + # wait for checkpoint to be saved if hasattr(loop, "_put_checkpoint_fut"): await loop._put_checkpoint_fut + # schedule any new tasks if new_tasks := [t for t in loop.tasks.values() if not t.scheduled]: # send messages to executor - for task in new_tasks: - if task.scheduled: - continue - await self.producer.send( - self.topics.executor, - value=MessageToExecutor( - config=patch_configurable( - loop.config, - { - **loop.checkpoint_config["configurable"], - CONFIG_KEY_DEDUPE_TASKS: True, - }, + futures: list[asyncio.Future] = await asyncio.gather( + *( + self.producer.send( + self.topics.executor, + value=MessageToExecutor( + config=patch_configurable( + loop.config, + { + **loop.checkpoint_config["configurable"], + CONFIG_KEY_DEDUPE_TASKS: True, + }, + ), + task=ExecutorTask( + id=task.id, + path=task.path, + step=loop.step, + resuming=loop.input is INPUT_RESUMING, + ), ), - task=ExecutorTask( - id=task.id, - path=task.path, - step=loop.step, - resuming=loop.input is INPUT_RESUMING, - ), - ), + ) + for task in new_tasks ) - # flush producer - await self.producer.flush() + ) + # wait for messages to be sent + await asyncio.gather(*futures) # mark as scheduled for task in new_tasks: loop.put_writes(task.id, [(SCHEDULED, None)]) - # return message - return msg diff --git a/libs/scheduler-kafka/pyproject.toml b/libs/scheduler-kafka/pyproject.toml index d107afcc8..1799acb44 100644 --- a/libs/scheduler-kafka/pyproject.toml +++ b/libs/scheduler-kafka/pyproject.toml @@ -53,5 +53,5 @@ lint.ignore = ["E501", "B008", "UP007", "UP006"] [tool.pytest-watcher] now = true delay = 0.1 -runner_args = ["-s", "--ff", "-v", "--tb", "short"] +runner_args = ["--ff", "-v", "--tb", "short"] patterns = ["*.py"] diff --git a/libs/scheduler-kafka/tests/test_scheduler.py b/libs/scheduler-kafka/tests/test_scheduler.py index b53abd361..d4cda6210 100644 --- a/libs/scheduler-kafka/tests/test_scheduler.py +++ b/libs/scheduler-kafka/tests/test_scheduler.py @@ -1,6 +1,7 @@ import asyncio +import functools import operator -from typing import Annotated, TypedDict, Union +from typing import Annotated, Callable, ParamSpec, TypedDict, TypeVar, Union import pytest from aiokafka import AIOKafkaProducer @@ -14,6 +15,20 @@ from langgraph.scheduler.kafka.orchestrator import KafkaOrchestrator from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics pytestmark = pytest.mark.anyio +C = ParamSpec("C") +R = TypeVar("R") + + +def timeout(delay: int): + def decorator(func: Callable[C, R]) -> Callable[C, R]: + @functools.wraps(func) + async def new_func(*args: C.args, **kwargs: C.kwargs) -> R: + async with asyncio.timeout(delay): + return await func(*args, **kwargs) + + return new_func + + return decorator def mk_fanout_graph(checkpointer: BaseCheckpointSaver) -> Pregel: @@ -80,32 +95,32 @@ def mk_fanout_graph(checkpointer: BaseCheckpointSaver) -> Pregel: return builder.compile(checkpointer) +@timeout(5) async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) -> None: graph = mk_fanout_graph(checkpointer) n_orch_msgs = 0 n_exec_msgs = 0 - async def orchestrator() -> None: + async def orchestrator(expected: int) -> None: nonlocal n_orch_msgs async with KafkaOrchestrator(graph, topics) as orch: - async for msg in orch: - n_orch_msgs += 1 - print("orch", msg) + async for msgs in orch: + n_orch_msgs += len(msgs) + print("orch", msgs) + if n_orch_msgs == expected: + break - async def executor() -> None: + async def executor(expected: int) -> None: nonlocal n_exec_msgs async with KafkaExecutor(graph, topics) as exec: - async for msg in exec: - n_exec_msgs += 1 - print("exec", msg) + async for msgs in exec: + n_exec_msgs += len(msgs) + print("exec", msgs) + if n_exec_msgs == expected: + break - async with asyncio.TaskGroup() as tg: - o = tg.create_task(orchestrator(), name="orchestrator") - e = tg.create_task(executor(), name="executor") - - # start a new run - producer = AIOKafkaProducer(value_serializer=serde.dumps) - await producer.start() + # start a new run + async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: await producer.send_and_wait( topics.orchestrator, MessageToOrchestrator( @@ -113,12 +128,11 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - config={"configurable": {"thread_id": "1"}}, ), ) - await producer.stop() - # wait for the run to finish - await asyncio.sleep(5) - o.cancel() - e.cancel() + # run the orchestrator and executor + async with asyncio.TaskGroup() as tg: + tg.create_task(orchestrator(13), name="orchestrator") + tg.create_task(executor(12), name="executor") assert n_orch_msgs == 13 assert n_exec_msgs == 12 From f09b1308370b9673d1c95bd6b3738639bb5a95c3 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 12:56:55 -0700 Subject: [PATCH 05/34] Use pool for postgres checkpointer --- libs/scheduler-kafka/tests/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/scheduler-kafka/tests/conftest.py b/libs/scheduler-kafka/tests/conftest.py index 4296b9851..1d50ecc94 100644 --- a/libs/scheduler-kafka/tests/conftest.py +++ b/libs/scheduler-kafka/tests/conftest.py @@ -4,6 +4,7 @@ from uuid import uuid4 import kafka.admin import pytest from psycopg import AsyncConnection +from psycopg_pool import AsyncConnectionPool from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver from langgraph.scheduler.kafka.types import Topics @@ -45,9 +46,10 @@ async def checkpointer() -> AsyncIterator[AsyncPostgresSaver]: await conn.execute(f"CREATE DATABASE {database}") try: # yield checkpointer - async with AsyncPostgresSaver.from_conn_string( - DEFAULT_POSTGRES_URI + database - ) as checkpointer: + async with AsyncConnectionPool( + DEFAULT_POSTGRES_URI + database, max_size=10, kwargs={"autocommit": True} + ) as pool: + checkpointer = AsyncPostgresSaver(pool) await checkpointer.setup() yield checkpointer finally: From 5b3bd920d8d662251c719c711eacefba384ec01d Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 14:26:06 -0700 Subject: [PATCH 06/34] Catch errors thrown by nodes run by Executor --- libs/langgraph/langgraph/pregel/executor.py | 23 ++++++++++++------- libs/langgraph/langgraph/pregel/runner.py | 17 +++++++++++--- .../langgraph/scheduler/kafka/executor.py | 4 ++-- .../langgraph/scheduler/kafka/orchestrator.py | 2 +- libs/scheduler-kafka/pyproject.toml | 2 +- libs/scheduler-kafka/tests/test_scheduler.py | 12 +++++----- 6 files changed, 39 insertions(+), 21 deletions(-) diff --git a/libs/langgraph/langgraph/pregel/executor.py b/libs/langgraph/langgraph/pregel/executor.py index 981ebbf7d..e441c8f78 100644 --- a/libs/langgraph/langgraph/pregel/executor.py +++ b/libs/langgraph/langgraph/pregel/executor.py @@ -31,6 +31,7 @@ class Submit(Protocol[P, T]): *args: P.args, __name__: Optional[str] = None, __cancel_on_exit__: bool = False, + __reraise_on_exit__: bool = True, **kwargs: P.kwargs, ) -> concurrent.futures.Future[T]: ... @@ -39,7 +40,7 @@ class BackgroundExecutor(ContextManager): def __init__(self, config: RunnableConfig) -> None: self.stack = ExitStack() self.executor = self.stack.enter_context(get_executor_for_config(config)) - self.tasks: dict[concurrent.futures.Future, bool] = {} + self.tasks: dict[concurrent.futures.Future, tuple[bool, bool]] = {} def submit( self, @@ -47,10 +48,11 @@ class BackgroundExecutor(ContextManager): *args: P.args, __name__: Optional[str] = None, # currently not used in sync version __cancel_on_exit__: bool = False, + __reraise_on_exit__: bool = True, **kwargs: P.kwargs, ) -> concurrent.futures.Future[T]: task = self.executor.submit(fn, *args, **kwargs) - self.tasks[task] = __cancel_on_exit__ + self.tasks[task] = (__cancel_on_exit__, __reraise_on_exit__) task.add_done_callback(self.done) return task @@ -76,7 +78,7 @@ class BackgroundExecutor(ContextManager): traceback: Optional[TracebackType], ) -> Optional[bool]: # cancel all tasks that should be cancelled - for task, cancel in self.tasks.items(): + for task, (cancel, _) in self.tasks.items(): if cancel: task.cancel() # wait for all tasks to finish @@ -87,7 +89,9 @@ class BackgroundExecutor(ContextManager): # re-raise the first exception that occurred in a task if exc_type is None: # if there's already an exception being raised, don't raise another one - for task in self.tasks: + for task, (_, reraise) in self.tasks.items(): + if not reraise: + continue try: task.result() except concurrent.futures.CancelledError: @@ -97,7 +101,7 @@ class BackgroundExecutor(ContextManager): class AsyncBackgroundExecutor(AsyncContextManager): def __init__(self) -> None: self.context_not_supported = sys.version_info < (3, 11) - self.tasks: dict[asyncio.Task, bool] = {} + self.tasks: dict[asyncio.Task, tuple[bool, bool]] = {} self.sentinel = object() self.loop = asyncio.get_running_loop() @@ -107,6 +111,7 @@ class AsyncBackgroundExecutor(AsyncContextManager): *args: P.args, __name__: Optional[str] = None, __cancel_on_exit__: bool = False, + __reraise_on_exit__: bool = True, **kwargs: P.kwargs, ) -> asyncio.Task[T]: coro = fn(*args, **kwargs) @@ -114,7 +119,7 @@ class AsyncBackgroundExecutor(AsyncContextManager): task = self.loop.create_task(coro, name=__name__) else: task = self.loop.create_task(coro, name=__name__, context=copy_context()) - self.tasks[task] = __cancel_on_exit__ + self.tasks[task] = (__cancel_on_exit__, __reraise_on_exit__) task.add_done_callback(self.done) return task @@ -140,7 +145,7 @@ class AsyncBackgroundExecutor(AsyncContextManager): traceback: Optional[TracebackType], ) -> None: # cancel all tasks that should be cancelled - for task, cancel in self.tasks.items(): + for task, (cancel, _) in self.tasks.items(): if cancel: task.cancel(self.sentinel) # wait for all tasks to finish @@ -149,7 +154,9 @@ class AsyncBackgroundExecutor(AsyncContextManager): # if there's already an exception being raised, don't raise another one if exc_type is None: # re-raise the first exception that occurred in a task - for task in self.tasks: + for task, (_, reraise) in self.tasks.items(): + if not reraise: + continue try: if exc := task.exception(): raise exc diff --git a/libs/langgraph/langgraph/pregel/runner.py b/libs/langgraph/langgraph/pregel/runner.py index c895bf9e9..3f42fbd5f 100644 --- a/libs/langgraph/langgraph/pregel/runner.py +++ b/libs/langgraph/langgraph/pregel/runner.py @@ -35,6 +35,7 @@ class PregelRunner: self, tasks: list[PregelExecutableTask], *, + reraise: bool = True, timeout: Optional[float] = None, retry_policy: Optional[RetryPolicy] = None, ) -> Iterator[None]: @@ -48,6 +49,7 @@ class PregelRunner: run_with_retry, task, retry_policy, + __reraise_on_exit__=reraise, ): task for task in tasks if not task.writes @@ -84,12 +86,13 @@ class PregelRunner: # give control back to the caller yield # panic on failure or timeout - _panic_or_proceed(all_futures) + _panic_or_proceed(all_futures, panic=reraise) async def atick( self, tasks: list[PregelExecutableTask], *, + reraise: bool = True, timeout: Optional[float] = None, retry_policy: Optional[RetryPolicy] = None, ) -> AsyncIterator[None]: @@ -107,6 +110,7 @@ class PregelRunner: stream=self.use_astream, __name__=task.name, __cancel_on_exit__=True, + __reraise_on_exit__=reraise, ): task for task in tasks if not task.writes @@ -142,7 +146,9 @@ class PregelRunner: # give control back to the caller yield # panic on failure or timeout - _panic_or_proceed(all_futures, asyncio.TimeoutError) + _panic_or_proceed( + all_futures, timeout_exc_cls=asyncio.TimeoutError, panic=reraise + ) def _should_stop_others( @@ -171,7 +177,9 @@ def _exception( def _panic_or_proceed( futs: Union[set[concurrent.futures.Future[Any]], set[asyncio.Task[Any]]], + *, timeout_exc_cls: Type[Exception] = TimeoutError, + panic: bool = True, ) -> None: done: set[Union[concurrent.futures.Future[Any], asyncio.Task[Any]]] = set() inflight: set[Union[concurrent.futures.Future[Any], asyncio.Task[Any]]] = set() @@ -187,7 +195,10 @@ def _panic_or_proceed( while inflight: inflight.pop().cancel() # raise the exception - raise exc + if panic: + raise exc + else: + return if inflight: # if we got here means we timed out while inflight: diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index 82549cad9..1cf55266f 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -60,7 +60,7 @@ class KafkaExecutor(AbstractAsyncContextManager): return self async def __aexit__(self, *args: Any) -> None: - await self.stack.__aexit__(*args) + return await self.stack.__aexit__(*args) def __aiter__(self) -> Self: return self @@ -109,7 +109,7 @@ class KafkaExecutor(AbstractAsyncContextManager): submit=submit, put_writes=partial(self._put_writes, submit, msg["config"]), ) - async for _ in runner.atick([task]): + async for _ in runner.atick([task], reraise=False): pass else: # task was not found diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index b8fe1f003..408499f70 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -53,7 +53,7 @@ class KafkaOrchestrator(AbstractAsyncContextManager): return self async def __aexit__(self, *args: Any) -> None: - await self.stack.__aexit__(*args) + return await self.stack.__aexit__(*args) def __aiter__(self) -> Self: return self diff --git a/libs/scheduler-kafka/pyproject.toml b/libs/scheduler-kafka/pyproject.toml index 1799acb44..8ca9618e0 100644 --- a/libs/scheduler-kafka/pyproject.toml +++ b/libs/scheduler-kafka/pyproject.toml @@ -53,5 +53,5 @@ lint.ignore = ["E501", "B008", "UP007", "UP006"] [tool.pytest-watcher] now = true delay = 0.1 -runner_args = ["--ff", "-v", "--tb", "short"] +runner_args = ["--ff", "-v", "--tb", "short", "-s"] patterns = ["*.py"] diff --git a/libs/scheduler-kafka/tests/test_scheduler.py b/libs/scheduler-kafka/tests/test_scheduler.py index d4cda6210..b448c3c75 100644 --- a/libs/scheduler-kafka/tests/test_scheduler.py +++ b/libs/scheduler-kafka/tests/test_scheduler.py @@ -3,6 +3,7 @@ import functools import operator from typing import Annotated, Callable, ParamSpec, TypedDict, TypeVar, Union +import anyio import pytest from aiokafka import AIOKafkaProducer @@ -48,7 +49,6 @@ def mk_fanout_graph(checkpointer: BaseCheckpointSaver) -> Pregel: docs: Annotated[list[str], sorted_add] async def rewrite_query(data: State) -> State: - print("rewrite_query", data) return {"query": f'query: {data["query"]}'} async def retriever_picker(data: State) -> list[str]: @@ -105,8 +105,8 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - nonlocal n_orch_msgs async with KafkaOrchestrator(graph, topics) as orch: async for msgs in orch: - n_orch_msgs += len(msgs) print("orch", msgs) + n_orch_msgs += len(msgs) if n_orch_msgs == expected: break @@ -114,8 +114,8 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - nonlocal n_exec_msgs async with KafkaExecutor(graph, topics) as exec: async for msgs in exec: - n_exec_msgs += len(msgs) print("exec", msgs) + n_exec_msgs += len(msgs) if n_exec_msgs == expected: break @@ -130,9 +130,9 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - ) # run the orchestrator and executor - async with asyncio.TaskGroup() as tg: - tg.create_task(orchestrator(13), name="orchestrator") - tg.create_task(executor(12), name="executor") + async with anyio.create_task_group() as tg: + tg.start_soon(orchestrator, 13, name="orchestrator") + tg.start_soon(executor, 12, name="executor") assert n_orch_msgs == 13 assert n_exec_msgs == 12 From 2d02e5e0c5d2f5ef286b4ee90d1c02b4f9cdc0c5 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 15:18:13 -0700 Subject: [PATCH 07/34] Add error topic, add retries (eg for checkpointer database exceptions) --- .../langgraph/scheduler/kafka/executor.py | 20 ++++++- .../langgraph/scheduler/kafka/orchestrator.py | 20 ++++++- .../langgraph/scheduler/kafka/retry.py | 52 +++++++++++++++++++ .../langgraph/scheduler/kafka/types.py | 9 +++- libs/scheduler-kafka/tests/conftest.py | 6 ++- 5 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index 1cf55266f..c81455c50 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -1,7 +1,7 @@ import asyncio from contextlib import AbstractAsyncContextManager, AsyncExitStack from functools import partial -from typing import Any, Self, Sequence +from typing import Any, Optional, Self, Sequence import aiokafka from langchain_core.runnables import RunnableConfig @@ -14,7 +14,10 @@ from langgraph.pregel.algo import prepare_single_task from langgraph.pregel.executor import AsyncBackgroundExecutor, Submit from langgraph.pregel.manager import AsyncChannelsManager from langgraph.pregel.runner import PregelRunner +from langgraph.pregel.types import RetryPolicy +from langgraph.scheduler.kafka.retry import aretry from langgraph.scheduler.kafka.types import ( + ErrorMessage, MessageToExecutor, MessageToOrchestrator, Topics, @@ -30,6 +33,7 @@ class KafkaExecutor(AbstractAsyncContextManager): group_id: str = "executor", batch_max_n: int = 10, batch_max_ms: int = 1000, + retry_policy: Optional[RetryPolicy] = None, **kwargs: Any, ) -> None: self.graph = graph @@ -39,6 +43,7 @@ class KafkaExecutor(AbstractAsyncContextManager): self.group_id = group_id self.batch_max_n = batch_max_n self.batch_max_ms = batch_max_ms + self.retry_policy = retry_policy async def __aenter__(self) -> Self: self.consumer = await self.stack.enter_async_context( @@ -84,6 +89,19 @@ class KafkaExecutor(AbstractAsyncContextManager): return msgs async def each(self, msg: MessageToExecutor) -> None: + try: + await aretry(self.retry_policy, self.attempt, msg) + except Exception as exc: + await self.producer.send_and_wait( + self.topics.error, + value=ErrorMessage( + topic=self.topics.executor, + msg=msg, + error=repr(exc), + ), + ) + + async def attempt(self, msg: MessageToExecutor) -> None: # process message saved = await self.graph.checkpointer.aget_tuple(msg["config"]) if saved is None: diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index 408499f70..a8f41d99e 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -1,6 +1,6 @@ import asyncio from contextlib import AbstractAsyncContextManager, AsyncExitStack -from typing import Any, Self +from typing import Any, Optional, Self import aiokafka from langchain_core.runnables import ensure_config @@ -9,7 +9,10 @@ import langgraph.scheduler.kafka.serde as serde from langgraph.constants import CONFIG_KEY_DEDUPE_TASKS, SCHEDULED from langgraph.pregel import Pregel from langgraph.pregel.loop import INPUT_RESUMING, AsyncPregelLoop +from langgraph.pregel.types import RetryPolicy +from langgraph.scheduler.kafka.retry import aretry from langgraph.scheduler.kafka.types import ( + ErrorMessage, ExecutorTask, MessageToExecutor, MessageToOrchestrator, @@ -26,6 +29,7 @@ class KafkaOrchestrator(AbstractAsyncContextManager): group_id: str = "orchestrator", batch_max_n: int = 10, batch_max_ms: int = 1000, + retry_policy: Optional[RetryPolicy] = None, **kwargs: Any, ) -> None: self.graph = graph @@ -35,6 +39,7 @@ class KafkaOrchestrator(AbstractAsyncContextManager): self.group_id = group_id self.batch_max_n = batch_max_n self.batch_max_ms = batch_max_ms + self.retry_policy = retry_policy async def __aenter__(self) -> Self: self.consumer = await self.stack.enter_async_context( @@ -77,6 +82,19 @@ class KafkaOrchestrator(AbstractAsyncContextManager): return msgs async def each(self, msg: MessageToOrchestrator) -> None: + try: + await aretry(self.retry_policy, self.attempt, msg) + except Exception as exc: + await self.producer.send_and_wait( + self.topics.error, + value=ErrorMessage( + topic=self.topics.orchestrator, + msg=msg, + error=repr(exc), + ), + ) + + async def attempt(self, msg: MessageToOrchestrator) -> None: # process message async with AsyncPregelLoop( msg["input"], diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py new file mode 100644 index 000000000..52ad044b7 --- /dev/null +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py @@ -0,0 +1,52 @@ +import asyncio +import logging +import random +from typing import Awaitable, Callable, Optional, ParamSpec + +from langgraph.pregel.types import RetryPolicy + +logger = logging.getLogger(__name__) +P = ParamSpec("P") + + +async def aretry( + retry_policy: Optional[RetryPolicy], + func: Callable[P, Awaitable[None]], + *args: P.args, + **kwargs: P.kwargs, +) -> None: + """Run a task asynchronously with retries.""" + interval = retry_policy.initial_interval if retry_policy else 0 + attempts = 0 + while True: + try: + await func(*args, **kwargs) + # if successful, end + break + except Exception as exc: + if retry_policy is None: + raise + # increment attempts + attempts += 1 + # check if we should retry + if callable(retry_policy.retry_on): + if not retry_policy.retry_on(exc): + raise + elif not isinstance(exc, retry_policy.retry_on): + raise + # check if we should give up + if attempts >= retry_policy.max_attempts: + raise + # sleep before retrying + interval = min( + retry_policy.max_interval, + interval * retry_policy.backoff_factor, + ) + await asyncio.sleep( + interval + random.uniform(0, 1) if retry_policy.jitter else interval + ) + # log the retry + logger.info( + f"Retrying function {func} with {args} after {interval:.2f} seconds (attempt {attempts}) after {exc.__class__.__name__} {exc}", + exc_info=exc, + ) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py index ce2a01d9b..78396c88e 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py @@ -1,4 +1,4 @@ -from typing import Any, NamedTuple, Optional, TypedDict +from typing import Any, NamedTuple, Optional, TypedDict, Union from langchain_core.runnables import RunnableConfig @@ -6,6 +6,7 @@ from langchain_core.runnables import RunnableConfig class Topics(NamedTuple): orchestrator: str executor: str + error: str class MessageToOrchestrator(TypedDict): @@ -23,3 +24,9 @@ class ExecutorTask(TypedDict): class MessageToExecutor(TypedDict): config: RunnableConfig task: ExecutorTask + + +class ErrorMessage(TypedDict): + topic: str + error: str + msg: Union[MessageToExecutor, MessageToOrchestrator] diff --git a/libs/scheduler-kafka/tests/conftest.py b/libs/scheduler-kafka/tests/conftest.py index 1d50ecc94..7d855a113 100644 --- a/libs/scheduler-kafka/tests/conftest.py +++ b/libs/scheduler-kafka/tests/conftest.py @@ -21,18 +21,20 @@ def anyio_backend(): def topics() -> Iterator[Topics]: o = f"test_{uuid4().hex[:16]}" e = f"test_{uuid4().hex[:16]}" + z = f"test_{uuid4().hex[:16]}" admin = kafka.admin.KafkaAdminClient() # create topics admin.create_topics( [ kafka.admin.NewTopic(name=o, num_partitions=1, replication_factor=1), kafka.admin.NewTopic(name=e, num_partitions=1, replication_factor=1), + kafka.admin.NewTopic(name=z, num_partitions=1, replication_factor=1), ] ) # yield topics - yield Topics(orchestrator=o, executor=e) + yield Topics(orchestrator=o, executor=e, error=z) # delete topics - admin.delete_topics([o, e]) + admin.delete_topics([o, e, z]) admin.close() From ad463ec90a50836b800c58cafffb46208a21ffc2 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 15:53:16 -0700 Subject: [PATCH 08/34] Add more test assertions --- libs/scheduler-kafka/tests/test_scheduler.py | 25 ++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/libs/scheduler-kafka/tests/test_scheduler.py b/libs/scheduler-kafka/tests/test_scheduler.py index b448c3c75..fb9a47958 100644 --- a/libs/scheduler-kafka/tests/test_scheduler.py +++ b/libs/scheduler-kafka/tests/test_scheduler.py @@ -1,7 +1,7 @@ import asyncio import functools import operator -from typing import Annotated, Callable, ParamSpec, TypedDict, TypeVar, Union +from typing import Annotated, Callable, ParamSpec, Sequence, TypedDict, TypeVar, Union import anyio import pytest @@ -32,7 +32,9 @@ def timeout(delay: int): return decorator -def mk_fanout_graph(checkpointer: BaseCheckpointSaver) -> Pregel: +def mk_fanout_graph( + checkpointer: BaseCheckpointSaver, interrupt_before: Sequence[str] = () +) -> Pregel: # copied from test_in_one_fan_out_state_graph_waiting_edge_multiple_cond_edge def sorted_add( x: list[str], y: Union[list[str], list[tuple[str, str]]] @@ -92,11 +94,13 @@ def mk_fanout_graph(checkpointer: BaseCheckpointSaver) -> Pregel: builder.add_conditional_edges("decider", decider_cond) builder.set_finish_point("qa") - return builder.compile(checkpointer) + return builder.compile(checkpointer, interrupt_before=interrupt_before) @timeout(5) async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) -> None: + input = {"query": "what is weather in sf"} + config = {"configurable": {"thread_id": "1"}} graph = mk_fanout_graph(checkpointer) n_orch_msgs = 0 n_exec_msgs = 0 @@ -123,10 +127,7 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: await producer.send_and_wait( topics.orchestrator, - MessageToOrchestrator( - input={"query": "what is weather in sf"}, - config={"configurable": {"thread_id": "1"}}, - ), + MessageToOrchestrator(input=input, config=config), ) # run the orchestrator and executor @@ -134,5 +135,15 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - tg.start_soon(orchestrator, 13, name="orchestrator") tg.start_soon(executor, 12, name="executor") + state = await graph.aget_state(config) assert n_orch_msgs == 13 assert n_exec_msgs == 12 + assert ( + state.values + == await graph.ainvoke(input, {"configurable": {"thread_id": "2"}}) + == { + "docs": ["doc1", "doc1", "doc2", "doc2", "doc3", "doc3", "doc4", "doc4"], + "query": "analyzed: query: analyzed: query: what is weather in sf", + "answer": "doc1,doc1,doc2,doc2,doc3,doc3,doc4,doc4", + } + ) From 8e6caab78c739030ac2bc08e52a59d195e79d7b7 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 16:18:18 -0700 Subject: [PATCH 09/34] Add test for interrupts --- .../langgraph/scheduler/kafka/orchestrator.py | 6 +- libs/scheduler-kafka/tests/test_scheduler.py | 102 +++++++++++++++++- 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index a8f41d99e..6be0d2792 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -107,7 +107,11 @@ class KafkaOrchestrator(AbstractAsyncContextManager): output_keys=self.graph.output_channels, stream_keys=self.graph.stream_channels, ) as loop: - if loop.tick(input_keys=self.graph.input_channels): + if loop.tick( + input_keys=self.graph.input_channels, + interrupt_after=self.graph.interrupt_after_nodes, + interrupt_before=self.graph.interrupt_before_nodes, + ): # wait for checkpoint to be saved if hasattr(loop, "_put_checkpoint_fut"): await loop._put_checkpoint_fut diff --git a/libs/scheduler-kafka/tests/test_scheduler.py b/libs/scheduler-kafka/tests/test_scheduler.py index fb9a47958..00695e615 100644 --- a/libs/scheduler-kafka/tests/test_scheduler.py +++ b/libs/scheduler-kafka/tests/test_scheduler.py @@ -5,7 +5,7 @@ from typing import Annotated, Callable, ParamSpec, Sequence, TypedDict, TypeVar, import anyio import pytest -from aiokafka import AIOKafkaProducer +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer from langgraph.checkpoint.base import BaseCheckpointSaver from langgraph.graph.state import StateGraph @@ -97,7 +97,7 @@ def mk_fanout_graph( return builder.compile(checkpointer, interrupt_before=interrupt_before) -@timeout(5) +@timeout(10) async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) -> None: input = {"query": "what is weather in sf"} config = {"configurable": {"thread_id": "1"}} @@ -135,6 +135,12 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - tg.start_soon(orchestrator, 13, name="orchestrator") tg.start_soon(executor, 12, name="executor") + # check no errors + async with AIOKafkaConsumer(topics.error) as consumer: + assert len(consumer.assignment()) > 0 + for tp in consumer.assignment(): + assert await consumer.position(tp) == 0 + state = await graph.aget_state(config) assert n_orch_msgs == 13 assert n_exec_msgs == 12 @@ -147,3 +153,95 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - "answer": "doc1,doc1,doc2,doc2,doc3,doc3,doc4,doc4", } ) + + +@timeout(10) +async def test_fanout_graph_w_interrupt( + topics: Topics, checkpointer: BaseCheckpointSaver +) -> None: + input = {"query": "what is weather in sf"} + config = {"configurable": {"thread_id": "1"}} + graph = mk_fanout_graph(checkpointer, interrupt_before=["qa"]) + n_orch_msgs = 0 + n_exec_msgs = 0 + + async def orchestrator(expected: int) -> None: + nonlocal n_orch_msgs + async with KafkaOrchestrator(graph, topics) as orch: + async for msgs in orch: + print("orch", msgs) + n_orch_msgs += len(msgs) + if n_orch_msgs == expected: + break + + async def executor(expected: int) -> None: + nonlocal n_exec_msgs + async with KafkaExecutor(graph, topics) as exec: + async for msgs in exec: + print("exec", msgs) + n_exec_msgs += len(msgs) + if n_exec_msgs == expected: + break + + # start a new run + async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: + await producer.send_and_wait( + topics.orchestrator, + MessageToOrchestrator(input=input, config=config), + ) + + # run the orchestrator and executor + async with anyio.create_task_group() as tg: + tg.start_soon(orchestrator, 12, name="orchestrator") + tg.start_soon(executor, 11, name="executor") + + # check no errors + async with AIOKafkaConsumer(topics.error) as consumer: + assert len(consumer.assignment()) > 0 + for tp in consumer.assignment(): + assert await consumer.position(tp) == 0 + + state = await graph.aget_state(config) + assert n_orch_msgs == 12 + assert n_exec_msgs == 11 + assert state.next == ("qa",) + assert ( + state.values + == await graph.ainvoke(input, {"configurable": {"thread_id": "2"}}) + == { + "docs": ["doc1", "doc1", "doc2", "doc2", "doc3", "doc3", "doc4", "doc4"], + "query": "analyzed: query: analyzed: query: what is weather in sf", + } + ) + + # resume the thread + async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: + await producer.send_and_wait( + topics.orchestrator, + MessageToOrchestrator(input=None, config=config), + ) + + # run the orchestrator and executor + async with anyio.create_task_group() as tg: + tg.start_soon(orchestrator, 14, name="orchestrator") + tg.start_soon(executor, 12, name="executor") + + # check no errors + async with AIOKafkaConsumer(topics.error) as consumer: + assert len(consumer.assignment()) > 0 + for tp in consumer.assignment(): + assert await consumer.position(tp) == 0 + + state = await graph.aget_state(config) + assert n_orch_msgs == 14 + assert n_exec_msgs == 12 + assert state.next == () + assert ( + state.values + == await graph.ainvoke(None, {"configurable": {"thread_id": "2"}}) + == { + "answer": "doc1,doc1,doc2,doc2,doc3,doc3,doc4,doc4", + "docs": ["doc1", "doc1", "doc2", "doc2", "doc3", "doc3", "doc4", "doc4"], + "query": "analyzed: query: analyzed: query: what is weather in sf", + } + ) From d20662123002592b76cb9e936e7a75550d44dd33 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 17:48:55 -0700 Subject: [PATCH 10/34] Rename test --- .../scheduler-kafka/tests/{test_scheduler.py => test_fanout.py} | 2 ++ 1 file changed, 2 insertions(+) rename libs/scheduler-kafka/tests/{test_scheduler.py => test_fanout.py} (99%) diff --git a/libs/scheduler-kafka/tests/test_scheduler.py b/libs/scheduler-kafka/tests/test_fanout.py similarity index 99% rename from libs/scheduler-kafka/tests/test_scheduler.py rename to libs/scheduler-kafka/tests/test_fanout.py index 00695e615..ac8889045 100644 --- a/libs/scheduler-kafka/tests/test_scheduler.py +++ b/libs/scheduler-kafka/tests/test_fanout.py @@ -201,6 +201,7 @@ async def test_fanout_graph_w_interrupt( for tp in consumer.assignment(): assert await consumer.position(tp) == 0 + # check interrupted state state = await graph.aget_state(config) assert n_orch_msgs == 12 assert n_exec_msgs == 11 @@ -232,6 +233,7 @@ async def test_fanout_graph_w_interrupt( for tp in consumer.assignment(): assert await consumer.position(tp) == 0 + # check final state state = await graph.aget_state(config) assert n_orch_msgs == 14 assert n_exec_msgs == 12 From 83d36761a49f273717f5cc76e37e948491df18e4 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 17:49:28 -0700 Subject: [PATCH 11/34] Rename compose projects --- libs/langgraph/tests/compose-postgres.yml | 1 + libs/scheduler-kafka/tests/compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/libs/langgraph/tests/compose-postgres.yml b/libs/langgraph/tests/compose-postgres.yml index 80904ce90..221b35daf 100644 --- a/libs/langgraph/tests/compose-postgres.yml +++ b/libs/langgraph/tests/compose-postgres.yml @@ -1,3 +1,4 @@ +name: langgraph-tests services: postgres-test: image: postgres:16 diff --git a/libs/scheduler-kafka/tests/compose.yml b/libs/scheduler-kafka/tests/compose.yml index 59571371e..675769a0d 100644 --- a/libs/scheduler-kafka/tests/compose.yml +++ b/libs/scheduler-kafka/tests/compose.yml @@ -1,3 +1,4 @@ +name: scheduler-kafka-tests services: broker: image: apache/kafka:latest From b13db425689403ff08025940554312ceab831629 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Fri, 6 Sep 2024 18:20:06 -0700 Subject: [PATCH 12/34] WIP Subgraphs --- .../langgraph/checkpoint/postgres/aio.py | 8 +- .../langgraph/checkpoint/postgres/base.py | 10 + libs/langgraph/langgraph/pregel/loop.py | 16 +- .../langgraph/scheduler/kafka/executor.py | 1 + .../langgraph/scheduler/kafka/orchestrator.py | 19 +- libs/scheduler-kafka/tests/test_subgraph.py | 245 ++++++++++++++++++ 6 files changed, 295 insertions(+), 4 deletions(-) create mode 100644 libs/scheduler-kafka/tests/test_subgraph.py diff --git a/libs/checkpoint-postgres/langgraph/checkpoint/postgres/aio.py b/libs/checkpoint-postgres/langgraph/checkpoint/postgres/aio.py index 6140af756..488d39e99 100644 --- a/libs/checkpoint-postgres/langgraph/checkpoint/postgres/aio.py +++ b/libs/checkpoint-postgres/langgraph/checkpoint/postgres/aio.py @@ -10,6 +10,7 @@ from psycopg.types.json import Jsonb from psycopg_pool import AsyncConnectionPool from langgraph.checkpoint.base import ( + WRITES_IDX_MAP, ChannelVersions, Checkpoint, CheckpointMetadata, @@ -292,9 +293,14 @@ class AsyncPostgresSaver(BasePostgresSaver): writes (Sequence[Tuple[str, Any]]): List of writes to store, each as (channel, value) pair. task_id (str): Identifier for the task creating the writes. """ + query = ( + self.UPSERT_CHECKPOINT_WRITES_SQL + if all(w[0] in WRITES_IDX_MAP for w in writes) + else self.INSERT_CHECKPOINT_WRITES_SQL + ) async with self._cursor(pipeline=True) as cur: await cur.executemany( - self.UPSERT_CHECKPOINT_WRITES_SQL, + query, await asyncio.to_thread( self._dump_writes, config["configurable"]["thread_id"], diff --git a/libs/checkpoint-postgres/langgraph/checkpoint/postgres/base.py b/libs/checkpoint-postgres/langgraph/checkpoint/postgres/base.py index 6cfbc5108..0dd7d7733 100644 --- a/libs/checkpoint-postgres/langgraph/checkpoint/postgres/base.py +++ b/libs/checkpoint-postgres/langgraph/checkpoint/postgres/base.py @@ -108,6 +108,15 @@ UPSERT_CHECKPOINTS_SQL = """ """ UPSERT_CHECKPOINT_WRITES_SQL = """ + INSERT INTO checkpoint_writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, blob) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (thread_id, checkpoint_ns, checkpoint_id, task_id, idx) DO UPDATE SET + channel = EXCLUDED.channel, + type = EXCLUDED.type, + blob = EXCLUDED.blob; +""" + +INSERT_CHECKPOINT_WRITES_SQL = """ INSERT INTO checkpoint_writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, blob) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT (thread_id, checkpoint_ns, checkpoint_id, task_id, idx) DO NOTHING @@ -120,6 +129,7 @@ class BasePostgresSaver(BaseCheckpointSaver): UPSERT_CHECKPOINT_BLOBS_SQL = UPSERT_CHECKPOINT_BLOBS_SQL UPSERT_CHECKPOINTS_SQL = UPSERT_CHECKPOINTS_SQL UPSERT_CHECKPOINT_WRITES_SQL = UPSERT_CHECKPOINT_WRITES_SQL + INSERT_CHECKPOINT_WRITES_SQL = INSERT_CHECKPOINT_WRITES_SQL jsonplus_serde = JsonPlusSerializer() diff --git a/libs/langgraph/langgraph/pregel/loop.py b/libs/langgraph/langgraph/pregel/loop.py index c18fe5efb..d95ed9100 100644 --- a/libs/langgraph/langgraph/pregel/loop.py +++ b/libs/langgraph/langgraph/pregel/loop.py @@ -242,6 +242,14 @@ class PregelLoop: ) -> bool: """Execute a single iteration of the Pregel loop. Returns True if more iterations are needed.""" + print( + "tick", + self.config.get("configurable", {}).get("checkpoint_ns"), + self.step, + self.status, + self.input is INPUT_RESUMING, + self.input is INPUT_DONE, + ) if self.status != "pending": raise RuntimeError("Cannot tick when status is no longer 'pending'") @@ -345,7 +353,13 @@ class PregelLoop: continue if task := self.tasks.get(tid): if k == SCHEDULED: - self.tasks[tid] = task._replace(scheduled=True) + if v == max( + self.checkpoint["versions_seen"] + .get(INTERRUPT, {}) + .values(), + default=None, + ): + self.tasks[tid] = task._replace(scheduled=True) else: task.writes.append((k, v)) # print output for any tasks we applied previous writes to diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index c81455c50..e9fe4291b 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -147,4 +147,5 @@ class KafkaExecutor(AbstractAsyncContextManager): task_id: str, writes: list[tuple[str, Any]], ) -> None: + print("put_writes", task_id, writes) return submit(self.graph.checkpointer.aput_writes, config, writes, task_id) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index 6be0d2792..cb64f8f25 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -6,7 +6,7 @@ import aiokafka from langchain_core.runnables import ensure_config import langgraph.scheduler.kafka.serde as serde -from langgraph.constants import CONFIG_KEY_DEDUPE_TASKS, SCHEDULED +from langgraph.constants import CONFIG_KEY_DEDUPE_TASKS, INTERRUPT, SCHEDULED from langgraph.pregel import Pregel from langgraph.pregel.loop import INPUT_RESUMING, AsyncPregelLoop from langgraph.pregel.types import RetryPolicy @@ -145,4 +145,19 @@ class KafkaOrchestrator(AbstractAsyncContextManager): await asyncio.gather(*futures) # mark as scheduled for task in new_tasks: - loop.put_writes(task.id, [(SCHEDULED, None)]) + loop.put_writes( + task.id, + [ + ( + SCHEDULED, + max( + loop.checkpoint["versions_seen"] + .get(INTERRUPT, {}) + .values(), + default=None, + ), + ) + ], + ) + else: + pass diff --git a/libs/scheduler-kafka/tests/test_subgraph.py b/libs/scheduler-kafka/tests/test_subgraph.py new file mode 100644 index 000000000..15bb52ac2 --- /dev/null +++ b/libs/scheduler-kafka/tests/test_subgraph.py @@ -0,0 +1,245 @@ +import asyncio +import functools +import re +from typing import Callable, Literal, Optional, ParamSpec, TypeVar, Union, cast + +import anyio +import pytest +from aiokafka import AIOKafkaConsumer, AIOKafkaProducer +from langchain_core.language_models.fake_chat_models import ( + FakeMessagesListChatModel, +) +from langchain_core.messages import AIMessage, HumanMessage, ToolCall +from langchain_core.tools import tool + +from langgraph.checkpoint.base import BaseCheckpointSaver +from langgraph.constants import END, START +from langgraph.graph import MessagesState +from langgraph.graph.state import StateGraph +from langgraph.pregel import Pregel +from langgraph.scheduler.kafka import serde +from langgraph.scheduler.kafka.executor import KafkaExecutor +from langgraph.scheduler.kafka.orchestrator import KafkaOrchestrator +from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics + +pytestmark = pytest.mark.anyio +C = ParamSpec("C") +R = TypeVar("R") + + +class AnyStr(str): + def __init__(self, prefix: Union[str, re.Pattern] = "") -> None: + super().__init__() + self.prefix = prefix + + def __eq__(self, other: object) -> bool: + return isinstance(other, str) and ( + other.startswith(self.prefix) + if isinstance(self.prefix, str) + else self.prefix.match(other) + ) + + def __hash__(self) -> int: + return hash((str(self), self.prefix)) + + +def timeout(delay: int): + def decorator(func: Callable[C, R]) -> Callable[C, R]: + @functools.wraps(func) + async def new_func(*args: C.args, **kwargs: C.kwargs) -> R: + async with asyncio.timeout(delay): + return await func(*args, **kwargs) + + return new_func + + return decorator + + +def mk_weather_graph(checkpointer: BaseCheckpointSaver) -> Pregel: + # copied from test_weather_subgraph + + # setup subgraph + + @tool + def get_weather(city: str): + """Get the weather for a specific city""" + return f"I'ts sunny in {city}!" + + weather_model = FakeMessagesListChatModel( + responses=[ + AIMessage( + content="", + tool_calls=[ + ToolCall( + id="tool_call123", + name="get_weather", + args={"city": "San Francisco"}, + ) + ], + ) + ] + ) + + class SubGraphState(MessagesState): + city: str + + def model_node(state: SubGraphState): + result = weather_model.invoke(state["messages"]) + return {"city": cast(AIMessage, result).tool_calls[0]["args"]["city"]} + + def weather_node(state: SubGraphState): + result = get_weather.invoke({"city": state["city"]}) + return {"messages": [{"role": "assistant", "content": result}]} + + subgraph = StateGraph(SubGraphState) + subgraph.add_node(model_node) + subgraph.add_node(weather_node) + subgraph.add_edge(START, "model_node") + subgraph.add_edge("model_node", "weather_node") + subgraph.add_edge("weather_node", END) + subgraph = subgraph.compile(interrupt_before=["weather_node"]) + + # setup main graph + + class RouterState(MessagesState): + route: Literal["weather", "other"] + + router_model = FakeMessagesListChatModel( + responses=[ + AIMessage( + content="", + tool_calls=[ + ToolCall( + id="tool_call123", + name="router", + args={"dest": "weather"}, + ) + ], + ) + ] + ) + + def router_node(state: RouterState): + system_message = "Classify the incoming query as either about weather or not." + messages = [{"role": "system", "content": system_message}] + state["messages"] + route = router_model.invoke(messages) + return {"route": cast(AIMessage, route).tool_calls[0]["args"]["dest"]} + + def normal_llm_node(state: RouterState): + return {"messages": [AIMessage("Hello!")]} + + def route_after_prediction(state: RouterState): + if state["route"] == "weather": + return "weather_graph" + else: + return "normal_llm_node" + + async def weather_graph(state: RouterState): + return await subgraph.ainvoke(state) + + graph = StateGraph(RouterState) + graph.add_node(router_node) + graph.add_node(normal_llm_node) + graph.add_node("weather_graph", weather_graph) + graph.add_edge(START, "router_node") + graph.add_conditional_edges("router_node", route_after_prediction) + graph.add_edge("normal_llm_node", END) + graph.add_edge("weather_graph", END) + + return graph.compile(checkpointer=checkpointer) + + +@timeout(10) +async def test_subgraph_w_interrupt( + topics: Topics, checkpointer: BaseCheckpointSaver +) -> None: + input = {"messages": [{"role": "user", "content": "what's the weather in sf"}]} + config = {"configurable": {"thread_id": "1"}} + graph = mk_weather_graph(checkpointer) + n_orch_msgs = 0 + n_exec_msgs = 0 + errors = [] + scope: Optional[anyio.CancelScope] = None + + async def orchestrator(expected: int) -> None: + nonlocal n_orch_msgs + async with KafkaOrchestrator(graph, topics) as orch: + async for msgs in orch: + n_orch_msgs += len(msgs) + print("orch", n_orch_msgs, msgs) + if n_orch_msgs == expected: + break + + async def executor(expected: int) -> None: + nonlocal n_exec_msgs + async with KafkaExecutor(graph, topics) as exec: + async for msgs in exec: + n_exec_msgs += len(msgs) + print("exec", n_exec_msgs, msgs) + if n_exec_msgs == expected: + break + + async def error_consumer() -> None: + async with AIOKafkaConsumer(topics.error) as consumer: + async for msg in consumer: + errors.append(msg) + if scope: + scope.cancel() + + # start error consumer + error_task = asyncio.create_task(error_consumer(), name="error_consumer") + + # start a new run + async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: + await producer.send_and_wait( + topics.orchestrator, + MessageToOrchestrator(input=input, config=config), + ) + + # run the orchestrator and executor + async with anyio.create_task_group() as tg: + scope = tg.cancel_scope + tg.start_soon(orchestrator, 4, name="orchestrator") + tg.start_soon(executor, 3, name="executor") + + # check no errors + assert not errors + + # check interrupted state + state = await graph.aget_state(config) + assert n_orch_msgs == 4 + assert n_exec_msgs == 3 + assert state.next == ("weather_graph",) + assert state.values == { + "messages": [HumanMessage(id=AnyStr(), content="what's the weather in sf")], + "route": "weather", + } + + # resume the thread + async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: + await producer.send_and_wait( + topics.orchestrator, + MessageToOrchestrator(input=None, config=config), + ) + + # run the orchestrator and executor + async with anyio.create_task_group() as tg: + scope = tg.cancel_scope + tg.start_soon(orchestrator, 6, name="orchestrator") + tg.start_soon(executor, 4, name="executor") + + # check no errors + assert not errors + + # check final state + state = await graph.aget_state(config) + assert n_orch_msgs == 6 + assert n_exec_msgs == 4 + assert state.next == () + assert state.values == { + "answer": "doc1,doc1,doc2,doc2,doc3,doc3,doc4,doc4", + "docs": ["doc1", "doc1", "doc2", "doc2", "doc3", "doc3", "doc4", "doc4"], + "query": "analyzed: query: analyzed: query: what is weather in sf", + } + + error_task.cancel() From 5e6b58f8476f30db12e74e16b815c5b4b319a9af Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Mon, 9 Sep 2024 08:50:38 -0700 Subject: [PATCH 13/34] Remove double shield usage - we already shield from cancellation in PregelLoop so shouldnt be repeating in AsyncBackgroundExecutor --- libs/langgraph/langgraph/pregel/executor.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/libs/langgraph/langgraph/pregel/executor.py b/libs/langgraph/langgraph/pregel/executor.py index e441c8f78..f606d4c28 100644 --- a/libs/langgraph/langgraph/pregel/executor.py +++ b/libs/langgraph/langgraph/pregel/executor.py @@ -138,7 +138,7 @@ class AsyncBackgroundExecutor(AsyncContextManager): async def __aenter__(self) -> Submit: return self.submit - async def exit( + async def __aexit__( self, exc_type: Optional[type[BaseException]], exc_value: Optional[BaseException], @@ -162,15 +162,3 @@ class AsyncBackgroundExecutor(AsyncContextManager): raise exc except asyncio.CancelledError: pass - - async def __aexit__( - self, - exc_type: Optional[type[BaseException]], - exc_value: Optional[BaseException], - traceback: Optional[TracebackType], - ) -> Optional[bool]: - # we cannot use `await` outside of asyncio.shield, as this code can run - # after owning task is cancelled, so pulling async logic to separate method - - # wait for all background tasks to finish, shielded from cancellation - await asyncio.shield(self.exit(exc_type, exc_value, traceback)) From 1ce8d628f248b0bf90c5d8b4f2c02602cfd189af Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Mon, 9 Sep 2024 09:17:36 -0700 Subject: [PATCH 14/34] In update state skip nodes not found in instance eg, because removed since --- libs/langgraph/langgraph/pregel/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/langgraph/langgraph/pregel/__init__.py b/libs/langgraph/langgraph/pregel/__init__.py index 8c7a05e13..d1229d6e9 100644 --- a/libs/langgraph/langgraph/pregel/__init__.py +++ b/libs/langgraph/langgraph/pregel/__init__.py @@ -782,6 +782,7 @@ class Pregel(Runnable[Union[dict[str, Any], Any], Union[dict[str, Any], Any]]): last_seen_by_node = sorted( (v, n) for n, seen in checkpoint["versions_seen"].items() + if n in self.nodes for v in seen.values() ) # if two nodes updated the state at the same time, it's ambiguous @@ -939,6 +940,7 @@ class Pregel(Runnable[Union[dict[str, Any], Any], Union[dict[str, Any], Any]]): last_seen_by_node = sorted( (v, n) for n, seen in checkpoint["versions_seen"].items() + if n in self.nodes for v in seen.values() ) # if two nodes updated the state at the same time, it's ambiguous From 0eb11a3cf9fb8b367748380f555f39c6670236c9 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Mon, 9 Sep 2024 16:17:26 -0700 Subject: [PATCH 15/34] Remove is_resuming arg --- libs/langgraph/langgraph/constants.py | 8 +- .../langgraph/managed/is_last_step.py | 2 +- libs/langgraph/langgraph/pregel/algo.py | 30 +-- libs/langgraph/langgraph/pregel/debug.py | 1 + libs/langgraph/langgraph/pregel/loop.py | 23 +- libs/langgraph/langgraph/pregel/types.py | 1 + libs/langgraph/tests/test_pregel.py | 211 +++++++++++------- libs/langgraph/tests/test_pregel_async.py | 179 ++++++++++----- .../langgraph/scheduler/kafka/executor.py | 3 +- .../langgraph/scheduler/kafka/orchestrator.py | 16 +- .../langgraph/scheduler/kafka/types.py | 2 - 11 files changed, 286 insertions(+), 190 deletions(-) diff --git a/libs/langgraph/langgraph/constants.py b/libs/langgraph/langgraph/constants.py index 9cc287277..7cea9de85 100644 --- a/libs/langgraph/langgraph/constants.py +++ b/libs/langgraph/langgraph/constants.py @@ -15,15 +15,17 @@ CONFIG_KEY_CHECKPOINT_MAP = "checkpoint_map" INTERRUPT = "__interrupt__" ERROR = "__error__" SCHEDULED = "__scheduled__" -TASKS = "__pregel_tasks" -SUBSCRIPTIONS = "__pregel_subscriptions" +TASKS = "__pregel_tasks" # for backwards compat, this is the original name of PUSH +PUSH = "__pregel_push" +PULL = "__pregel_pull" RUNTIME_PLACEHOLDER = "__pregel_runtime_placeholder__" RESERVED = { SCHEDULED, INTERRUPT, ERROR, TASKS, - SUBSCRIPTIONS, + PUSH, + PULL, CONFIG_KEY_SEND, CONFIG_KEY_READ, CONFIG_KEY_CHECKPOINTER, diff --git a/libs/langgraph/langgraph/managed/is_last_step.py b/libs/langgraph/langgraph/managed/is_last_step.py index 3c6daf5f4..d8b4f9102 100644 --- a/libs/langgraph/langgraph/managed/is_last_step.py +++ b/libs/langgraph/langgraph/managed/is_last_step.py @@ -5,7 +5,7 @@ from langgraph.managed.base import ManagedValue class IsLastStepManager(ManagedValue[bool]): def __call__(self, step: int) -> bool: - return step == self.config["recursion_limit"] - 1 + return step == self.config.get("recursion_limit", 0) - 1 IsLastStep = Annotated[bool, IsLastStepManager] diff --git a/libs/langgraph/langgraph/pregel/algo.py b/libs/langgraph/langgraph/pregel/algo.py index 033fab8e0..2f947be22 100644 --- a/libs/langgraph/langgraph/pregel/algo.py +++ b/libs/langgraph/langgraph/pregel/algo.py @@ -25,13 +25,13 @@ from langgraph.constants import ( CONFIG_KEY_CHECKPOINT_MAP, CONFIG_KEY_CHECKPOINTER, CONFIG_KEY_READ, - CONFIG_KEY_RESUMING, CONFIG_KEY_SEND, CONFIG_KEY_TASK_ID, INTERRUPT, NS_SEP, + PULL, + PUSH, RESERVED, - SUBSCRIPTIONS, TAG_HIDDEN, TASKS, Send, @@ -242,7 +242,6 @@ def prepare_next_tasks( step: int, *, for_execution: Literal[False], - is_resuming: bool = False, checkpointer: Literal[None] = None, manager: Literal[None] = None, ) -> dict[str, PregelTask]: ... @@ -258,7 +257,6 @@ def prepare_next_tasks( step: int, *, for_execution: Literal[True], - is_resuming: bool, checkpointer: Optional[BaseCheckpointSaver], manager: Union[None, ParentRunManager, AsyncParentRunManager], ) -> dict[str, PregelExecutableTask]: ... @@ -273,7 +271,6 @@ def prepare_next_tasks( step: int, *, for_execution: bool, - is_resuming: bool = False, checkpointer: Optional[BaseCheckpointSaver] = None, manager: Union[None, ParentRunManager, AsyncParentRunManager] = None, ) -> Union[dict[str, PregelTask], dict[str, PregelExecutableTask]]: @@ -281,7 +278,7 @@ def prepare_next_tasks( # Consume pending packets for idx, _ in enumerate(checkpoint["pending_sends"]): if task := prepare_single_task( - (TASKS, idx), + (PUSH, idx), None, checkpoint=checkpoint, processes=processes, @@ -290,7 +287,6 @@ def prepare_next_tasks( config=config, step=step, for_execution=for_execution, - is_resuming=is_resuming, checkpointer=checkpointer, manager=manager, ): @@ -299,7 +295,7 @@ def prepare_next_tasks( # If so, prepare the values to be passed to them for name in processes: if task := prepare_single_task( - (SUBSCRIPTIONS, name), + (PULL, name), None, checkpoint=checkpoint, processes=processes, @@ -308,7 +304,6 @@ def prepare_next_tasks( config=config, step=step, for_execution=for_execution, - is_resuming=is_resuming, checkpointer=checkpointer, manager=manager, ): @@ -327,7 +322,6 @@ def prepare_single_task( config: RunnableConfig, step: int, for_execution: bool, - is_resuming: bool = False, checkpointer: Optional[BaseCheckpointSaver] = None, manager: Union[None, ParentRunManager, AsyncParentRunManager] = None, ) -> Union[None, PregelTask, PregelExecutableTask]: @@ -335,7 +329,7 @@ def prepare_single_task( configurable = config.get("configurable", {}) parent_ns = configurable.get("checkpoint_ns", "") - if task_path[0] == TASKS: + if task_path[0] == PUSH: idx = int(task_path[1]) packet = checkpoint["pending_sends"][idx] if not isinstance(packet, Send): @@ -347,7 +341,7 @@ def prepare_single_task( logger.warning(f"Ignoring unknown node name {packet.node} in pending sends") return # create task id - triggers = [TASKS] + triggers = [PUSH] metadata = { "langgraph_step": step, "langgraph_node": packet.node, @@ -362,7 +356,7 @@ def prepare_single_task( checkpoint_ns, str(step), packet.node, - TASKS, + PUSH, str(idx), ) if task_id_checksum is not None: @@ -416,7 +410,6 @@ def prepare_single_task( **configurable.get(CONFIG_KEY_CHECKPOINT_MAP, {}), parent_ns: checkpoint["id"], }, - CONFIG_KEY_RESUMING: is_resuming, "checkpoint_id": None, "checkpoint_ns": task_checkpoint_ns, }, @@ -429,8 +422,8 @@ def prepare_single_task( ) else: - return PregelTask(task_id, packet.node) - elif task_path[0] == SUBSCRIPTIONS: + return PregelTask(task_id, packet.node, task_path) + elif task_path[0] == PULL: name = str(task_path[1]) proc = processes[name] version_type = type(next(iter(checkpoint["channel_versions"].values()), None)) @@ -470,7 +463,7 @@ def prepare_single_task( checkpoint_ns, str(step), name, - SUBSCRIPTIONS, + PULL, *triggers, ) if task_id_checksum is not None: @@ -525,7 +518,6 @@ def prepare_single_task( **configurable.get(CONFIG_KEY_CHECKPOINT_MAP, {}), parent_ns: checkpoint["id"], }, - CONFIG_KEY_RESUMING: is_resuming, "checkpoint_ns": task_checkpoint_ns, }, ), @@ -536,7 +528,7 @@ def prepare_single_task( task_path, ) else: - return PregelTask(task_id, name) + return PregelTask(task_id, name, task_path) def _proc_input( diff --git a/libs/langgraph/langgraph/pregel/debug.py b/libs/langgraph/langgraph/pregel/debug.py index cb6e45e0d..a83c95709 100644 --- a/libs/langgraph/langgraph/pregel/debug.py +++ b/libs/langgraph/langgraph/pregel/debug.py @@ -210,6 +210,7 @@ def tasks_w_writes( PregelTask( task.id, task.name, + task.path, next( ( exc diff --git a/libs/langgraph/langgraph/pregel/loop.py b/libs/langgraph/langgraph/pregel/loop.py index d95ed9100..7772827c4 100644 --- a/libs/langgraph/langgraph/pregel/loop.py +++ b/libs/langgraph/langgraph/pregel/loop.py @@ -191,7 +191,7 @@ class PregelLoop: ) if not self.is_nested and config["configurable"].get("checkpoint_ns"): self.config = patch_configurable( - config, {"checkpoint_ns": "", "checkpoint_id": None} + self.config, {"checkpoint_ns": "", "checkpoint_id": None} ) if ( CONFIG_KEY_CHECKPOINT_MAP in self.config["configurable"] @@ -242,15 +242,6 @@ class PregelLoop: ) -> bool: """Execute a single iteration of the Pregel loop. Returns True if more iterations are needed.""" - print( - "tick", - self.config.get("configurable", {}).get("checkpoint_ns"), - self.step, - self.status, - self.input is INPUT_RESUMING, - self.input is INPUT_DONE, - ) - if self.status != "pending": raise RuntimeError("Cannot tick when status is no longer 'pending'") @@ -322,7 +313,6 @@ class PregelLoop: for_execution=True, manager=manager, checkpointer=self.checkpointer, - is_resuming=self.input is INPUT_RESUMING, ) # produce debug output @@ -402,9 +392,9 @@ class PregelLoop: # resuming from previous checkpoint requires # - finding a previous checkpoint # - receiving None input (outer graph) or RESUMING flag (subgraph) + configurable = self.config.get("configurable", {}) is_resuming = bool(self.checkpoint["channel_versions"]) and bool( - self.config.get("configurable", {}).get(CONFIG_KEY_RESUMING) - or self.input is None + configurable.get(CONFIG_KEY_RESUMING, self.input is None) ) # proceed past previous checkpoint @@ -441,10 +431,15 @@ class PregelLoop: ), "Can't write to SharedValues in graph input" # save input checkpoint self._put_checkpoint({"source": "input", "writes": dict(input_writes)}) - else: + elif CONFIG_KEY_RESUMING not in configurable: raise EmptyInputError(f"Received no input for {input_keys}") # done with input self.input = INPUT_RESUMING if is_resuming else INPUT_DONE + # update config + if not self.is_nested: + self.config = patch_configurable( + self.config, {CONFIG_KEY_RESUMING: is_resuming} + ) def _put_checkpoint(self, metadata: CheckpointMetadata) -> None: # assign step diff --git a/libs/langgraph/langgraph/pregel/types.py b/libs/langgraph/langgraph/pregel/types.py index 80d0d8a23..589ed43ab 100644 --- a/libs/langgraph/langgraph/pregel/types.py +++ b/libs/langgraph/langgraph/pregel/types.py @@ -66,6 +66,7 @@ class CachePolicy(NamedTuple): class PregelTask(NamedTuple): id: str name: str + path: tuple[str, ...] error: Optional[Exception] = None interrupts: tuple[Interrupt, ...] = () state: Union[None, RunnableConfig, "StateSnapshot"] = None diff --git a/libs/langgraph/tests/test_pregel.py b/libs/langgraph/tests/test_pregel.py index 335ccc2a5..ec154a2c7 100644 --- a/libs/langgraph/tests/test_pregel.py +++ b/libs/langgraph/tests/test_pregel.py @@ -52,7 +52,7 @@ from langgraph.checkpoint.base import ( CheckpointTuple, ) from langgraph.checkpoint.memory import MemorySaver -from langgraph.constants import ERROR, Interrupt, Send +from langgraph.constants import ERROR, PULL, PUSH, Interrupt, Send from langgraph.errors import InvalidUpdateError, NodeInterrupt from langgraph.graph import END, Graph from langgraph.graph.graph import START @@ -741,7 +741,7 @@ def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 4, "output": 4, "input": 3}, - tasks=(PregelTask(AnyStr(), "two"),), + tasks=(PregelTask(AnyStr(), "two", (PULL, "two")),), next=("two",), config={ "configurable": { @@ -761,7 +761,7 @@ def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 21, "output": 4, "input": 3}, - tasks=(PregelTask(AnyStr(), "one"),), + tasks=(PregelTask(AnyStr(), "one", (PULL, "one")),), next=("one",), config={ "configurable": { @@ -781,7 +781,7 @@ def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 21, "output": 4, "input": 20}, - tasks=(PregelTask(AnyStr(), "two"),), + tasks=(PregelTask(AnyStr(), "two", (PULL, "two")),), next=("two",), config={ "configurable": { @@ -801,7 +801,7 @@ def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 3, "output": 4, "input": 20}, - tasks=(PregelTask(AnyStr(), "one"),), + tasks=(PregelTask(AnyStr(), "one", (PULL, "one")),), next=("one",), config={ "configurable": { @@ -836,7 +836,7 @@ def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 3, "input": 2}, - tasks=(PregelTask(AnyStr(), "two"),), + tasks=(PregelTask(AnyStr(), "two", (PULL, "two")),), next=("two",), config={ "configurable": { @@ -856,7 +856,7 @@ def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"input": 2}, - tasks=(PregelTask(AnyStr(), "one"),), + tasks=(PregelTask(AnyStr(), "one", (PULL, "one")),), next=("one",), config={ "configurable": { @@ -941,7 +941,7 @@ def test_fork_always_re_runs_nodes( ), StateSnapshot( values=5, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -961,7 +961,7 @@ def test_fork_always_re_runs_nodes( ), StateSnapshot( values=4, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -981,7 +981,7 @@ def test_fork_always_re_runs_nodes( ), StateSnapshot( values=3, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -1001,7 +1001,7 @@ def test_fork_always_re_runs_nodes( ), StateSnapshot( values=2, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -1021,7 +1021,7 @@ def test_fork_always_re_runs_nodes( ), StateSnapshot( values=1, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -1036,7 +1036,7 @@ def test_fork_always_re_runs_nodes( ), StateSnapshot( values=0, - tasks=(PregelTask(AnyStr(), "__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), next=("__start__",), config={ "configurable": { @@ -1474,8 +1474,8 @@ def test_pending_writes_resume( assert state.values == {"value": 1} assert state.next == ("one", "two") assert state.tasks == ( - PregelTask(AnyStr(), "one"), - PregelTask(AnyStr(), "two", 'ConnectionError("I\'m not good")'), + PregelTask(AnyStr(), "one", (PULL, "one")), + PregelTask(AnyStr(), "two", (PULL, "two"), 'ConnectionError("I\'m not good")'), ) assert state.metadata == { "parents": {}, @@ -2261,7 +2261,7 @@ def test_conditional_graph( ), }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], config=app_w_interrupt.checkpointer.get_tuple(config).config, @@ -2314,7 +2314,7 @@ def test_conditional_graph( "input": "what is weather in sf", }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -2487,7 +2487,7 @@ def test_conditional_graph( ), }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -2534,7 +2534,7 @@ def test_conditional_graph( "input": "what is weather in sf", }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -2707,7 +2707,7 @@ def test_conditional_graph( ), }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3182,7 +3182,7 @@ def test_conditional_state_graph( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3224,7 +3224,7 @@ def test_conditional_state_graph( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3349,7 +3349,7 @@ def test_conditional_state_graph( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3390,7 +3390,7 @@ def test_conditional_state_graph( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3501,7 +3501,7 @@ def test_conditional_state_graph( values={ "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "agent"),), + tasks=(PregelTask(AnyStr(), "agent", (PULL, "agent")),), next=("agent",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3526,7 +3526,7 @@ def test_conditional_state_graph( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3580,7 +3580,7 @@ def test_conditional_state_graph( ] ], }, - tasks=(PregelTask(AnyStr(), "agent"),), + tasks=(PregelTask(AnyStr(), "agent", (PULL, "agent")),), next=("agent",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3645,7 +3645,7 @@ def test_conditional_state_graph( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -3699,7 +3699,7 @@ def test_conditional_state_graph( ] ], }, - tasks=(PregelTask(AnyStr(), "agent"),), + tasks=(PregelTask(AnyStr(), "agent", (PULL, "agent")),), next=("agent",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -4482,7 +4482,7 @@ def test_state_graph_packets( ), ] }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PUSH, 0)),), next=("tools",), config=(app_w_interrupt.checkpointer.get_tuple(config)).config, created_at=(app_w_interrupt.checkpointer.get_tuple(config)).checkpoint["ts"], @@ -4534,7 +4534,7 @@ def test_state_graph_packets( ), ] }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PUSH, 0)),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=(app_w_interrupt.checkpointer.get_tuple(config)).checkpoint["ts"], @@ -4632,7 +4632,10 @@ def test_state_graph_packets( ), ] }, - tasks=(PregelTask(AnyStr(), "tools"), PregelTask(AnyStr(), "tools")), + tasks=( + PregelTask(AnyStr(), "tools", (PUSH, 0)), + PregelTask(AnyStr(), "tools", (PUSH, 1)), + ), next=("tools", "tools"), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=(app_w_interrupt.checkpointer.get_tuple(config)).checkpoint["ts"], @@ -4977,7 +4980,7 @@ def test_message_graph( id="ai1", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5023,7 +5026,7 @@ def test_message_graph( ], ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=next_config, created_at=AnyStr(), @@ -5104,7 +5107,7 @@ def test_message_graph( id="ai2", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5207,7 +5210,7 @@ def test_message_graph( id="ai1", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5253,7 +5256,7 @@ def test_message_graph( ], ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5334,7 +5337,7 @@ def test_message_graph( id="ai2", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5428,7 +5431,7 @@ def test_message_graph( AIMessage(content="answer", id="ai2"), _AnyIdAIMessage(content="an extra message"), ], - tasks=(PregelTask(AnyStr(), "agent"),), + tasks=(PregelTask(AnyStr(), "agent", (PULL, "agent")),), next=("agent",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5703,7 +5706,7 @@ def test_root_graph( id="ai1", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5749,7 +5752,7 @@ def test_root_graph( ], ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=next_config, created_at=AnyStr(), @@ -5831,7 +5834,7 @@ def test_root_graph( id="ai2", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5935,7 +5938,7 @@ def test_root_graph( id="ai1", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -5981,7 +5984,7 @@ def test_root_graph( ], ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -6063,7 +6066,7 @@ def test_root_graph( id="ai2", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -6156,7 +6159,7 @@ def test_root_graph( AIMessage(content="answer", id="ai2"), _AnyIdAIMessage(content="an extra message"), ], - tasks=(PregelTask(AnyStr(), "agent"),), + tasks=(PregelTask(AnyStr(), "agent", (PULL, "agent")),), next=("agent",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -6228,7 +6231,7 @@ def test_root_graph( _AnyIdAIMessage(content="an extra message"), ] }, - tasks=(PregelTask(AnyStr(), "agent"),), + tasks=(PregelTask(AnyStr(), "agent", (PULL, "agent")),), next=("agent",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -6581,6 +6584,7 @@ def test_dynamic_interrupt( PregelTask( AnyStr(), "tool_two", + (PULL, "tool_two"), interrupts=(Interrupt("Just because..."),), ), ), @@ -6673,7 +6677,7 @@ def test_start_branch_then( ] assert tool_two.get_state(thread1) == StateSnapshot( values={"my_key": "value ⛰️", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "tool_two_slow"),), + tasks=(PregelTask(AnyStr(), "tool_two_slow", (PULL, "tool_two_slow")),), next=("tool_two_slow",), config=tool_two.checkpointer.get_tuple(thread1).config, created_at=tool_two.checkpointer.get_tuple(thread1).checkpoint["ts"], @@ -6708,7 +6712,7 @@ def test_start_branch_then( } assert tool_two.get_state(thread2) == StateSnapshot( values={"my_key": "value", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=tool_two.checkpointer.get_tuple(thread2).config, created_at=tool_two.checkpointer.get_tuple(thread2).checkpoint["ts"], @@ -6743,7 +6747,7 @@ def test_start_branch_then( } assert tool_two.get_state(thread3) == StateSnapshot( values={"my_key": "value", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=tool_two.checkpointer.get_tuple(thread3).config, created_at=tool_two.checkpointer.get_tuple(thread3).checkpoint["ts"], @@ -6754,7 +6758,7 @@ def test_start_branch_then( tool_two.update_state(thread3, {"my_key": "key"}) # appends to my_key assert tool_two.get_state(thread3) == StateSnapshot( values={"my_key": "valuekey", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=tool_two.checkpointer.get_tuple(thread3).config, created_at=tool_two.checkpointer.get_tuple(thread3).checkpoint["ts"], @@ -7064,7 +7068,7 @@ def test_branch_then( } assert tool_two.get_state(thread1) == StateSnapshot( values={"my_key": "value prepared", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "tool_two_slow"),), + tasks=(PregelTask(AnyStr(), "tool_two_slow", (PULL, "tool_two_slow")),), next=("tool_two_slow",), config=tool_two.checkpointer.get_tuple(thread1).config, created_at=tool_two.checkpointer.get_tuple(thread1).checkpoint["ts"], @@ -7104,7 +7108,7 @@ def test_branch_then( } assert tool_two.get_state(thread2) == StateSnapshot( values={"my_key": "value prepared", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=tool_two.checkpointer.get_tuple(thread2).config, created_at=tool_two.checkpointer.get_tuple(thread2).checkpoint["ts"], @@ -7152,7 +7156,7 @@ def test_branch_then( "my_key": "value prepared slow", "market": "DE", }, - tasks=(PregelTask(AnyStr(), "finish"),), + tasks=(PregelTask(AnyStr(), "finish", (PULL, "finish")),), next=("finish",), config=tool_two.checkpointer.get_tuple(thread1).config, created_at=tool_two.checkpointer.get_tuple(thread1).checkpoint["ts"], @@ -7172,7 +7176,7 @@ def test_branch_then( "my_key": "value prepared slower", "market": "DE", }, - tasks=(PregelTask(AnyStr(), "finish"),), + tasks=(PregelTask(AnyStr(), "finish", (PULL, "finish")),), next=("finish",), config=tool_two.checkpointer.get_tuple(thread1).config, created_at=tool_two.checkpointer.get_tuple(thread1).checkpoint["ts"], @@ -7201,7 +7205,7 @@ def test_branch_then( } assert tool_two.get_state(thread1) == StateSnapshot( values={"my_key": "value prepared", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "tool_two_slow"),), + tasks=(PregelTask(AnyStr(), "tool_two_slow", (PULL, "tool_two_slow")),), next=("tool_two_slow",), config=tool_two.checkpointer.get_tuple(thread1).config, created_at=tool_two.checkpointer.get_tuple(thread1).checkpoint["ts"], @@ -7241,7 +7245,7 @@ def test_branch_then( } assert tool_two.get_state(thread2) == StateSnapshot( values={"my_key": "value prepared", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=tool_two.checkpointer.get_tuple(thread2).config, created_at=tool_two.checkpointer.get_tuple(thread2).checkpoint["ts"], @@ -7279,7 +7283,7 @@ def test_branch_then( # check current state assert tool_two.get_state(thread3) == StateSnapshot( values={"my_key": "key", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "prepare"),), + tasks=(PregelTask(AnyStr(), "prepare", (PULL, "prepare")),), next=("prepare",), config=uconfig, created_at=AnyStr(), @@ -7299,7 +7303,7 @@ def test_branch_then( # get state after first node assert tool_two.get_state(thread3) == StateSnapshot( values={"my_key": "key prepared", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "tool_two_slow"),), + tasks=(PregelTask(AnyStr(), "tool_two_slow", (PULL, "tool_two_slow")),), next=("tool_two_slow",), config=tool_two.checkpointer.get_tuple(thread3).config, created_at=tool_two.checkpointer.get_tuple(thread3).checkpoint["ts"], @@ -7443,7 +7447,7 @@ def test_in_one_fan_out_state_graph_waiting_edge( "query": "analyzed: query: what is weather in sf", "docs": ["doc1", "doc2", "doc3", "doc4", "doc5"], }, - tasks=(PregelTask(AnyStr(), "qa"),), + tasks=(PregelTask(AnyStr(), "qa", (PULL, "qa")),), next=("qa",), config=app_w_interrupt.checkpointer.get_tuple(config).config, created_at=app_w_interrupt.checkpointer.get_tuple(config).checkpoint["ts"], @@ -8631,6 +8635,7 @@ def test_nested_graph_state( PregelTask( AnyStr(), "inner", + (PULL, "inner"), state={"configurable": {"thread_id": "1", "checkpoint_ns": AnyStr()}}, ), ), @@ -8664,6 +8669,7 @@ def test_nested_graph_state( PregelTask( AnyStr(), "inner", + (PULL, "inner"), state=StateSnapshot( values={ "my_key": "hi my value here", @@ -8672,8 +8678,8 @@ def test_nested_graph_state( tasks=( PregelTask( AnyStr(), - name="inner_2", - error=None, + "inner_2", + (PULL, "inner_2"), ), ), next=("inner_2",), @@ -8743,6 +8749,7 @@ def test_nested_graph_state( PregelTask( AnyStr(), "inner", + (PULL, "inner"), state={ "configurable": { "thread_id": "1", @@ -8776,7 +8783,7 @@ def test_nested_graph_state( ), StateSnapshot( values={"my_key": "my value"}, - tasks=(PregelTask(AnyStr(), "outer_1"),), + tasks=(PregelTask(AnyStr(), "outer_1", (PULL, "outer_1")),), next=("outer_1",), config={ "configurable": { @@ -8797,7 +8804,7 @@ def test_nested_graph_state( ), StateSnapshot( values={}, - tasks=(PregelTask(AnyStr(), "__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), next=("__start__",), config={ "configurable": { @@ -8851,7 +8858,7 @@ def test_nested_graph_state( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="inner_2"),), + tasks=(PregelTask(AnyStr(), "inner_2", (PULL, "inner_2")),), ), StateSnapshot( values={"my_key": "hi my value"}, @@ -8880,7 +8887,7 @@ def test_nested_graph_state( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="inner_1"),), + tasks=(PregelTask(AnyStr(), "inner_1", (PULL, "inner_1")),), ), StateSnapshot( values={}, @@ -8903,7 +8910,7 @@ def test_nested_graph_state( }, created_at=AnyStr(), parent_config=None, - tasks=(PregelTask(id=AnyStr(), name="__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), ), ] @@ -8971,7 +8978,7 @@ def test_nested_graph_state( ), StateSnapshot( values={"my_key": "hi my value here and there"}, - tasks=(PregelTask(AnyStr(), "outer_2"),), + tasks=(PregelTask(AnyStr(), "outer_2", (PULL, "outer_2")),), next=("outer_2",), config={ "configurable": { @@ -9001,6 +9008,7 @@ def test_nested_graph_state( PregelTask( AnyStr(), "inner", + (PULL, "inner"), state={ "configurable": {"thread_id": "1", "checkpoint_ns": AnyStr()} }, @@ -9031,7 +9039,7 @@ def test_nested_graph_state( ), StateSnapshot( values={"my_key": "my value"}, - tasks=(PregelTask(AnyStr(), "outer_1"),), + tasks=(PregelTask(AnyStr(), "outer_1", (PULL, "outer_1")),), next=("outer_1",), config={ "configurable": { @@ -9052,7 +9060,7 @@ def test_nested_graph_state( ), StateSnapshot( values={}, - tasks=(PregelTask(AnyStr(), "__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), next=("__start__",), config={ "configurable": { @@ -9149,6 +9157,7 @@ def test_doubly_nested_graph_state( PregelTask( AnyStr(), "child", + (PULL, "child"), state={ "configurable": { "thread_id": "1", @@ -9189,6 +9198,7 @@ def test_doubly_nested_graph_state( PregelTask( AnyStr(), "child_1", + (PULL, "child_1"), state={ "configurable": { "thread_id": "1", @@ -9228,6 +9238,7 @@ def test_doubly_nested_graph_state( PregelTask( AnyStr(), "grandchild_2", + (PULL, "grandchild_2"), ), ), next=("grandchild_2",), @@ -9272,18 +9283,21 @@ def test_doubly_nested_graph_state( PregelTask( AnyStr(), "child", + (PULL, "child"), state=StateSnapshot( values={"my_key": "hi my value"}, tasks=( PregelTask( AnyStr(), "child_1", + (PULL, "child_1"), state=StateSnapshot( values={"my_key": "hi my value here"}, tasks=( PregelTask( AnyStr(), "grandchild_2", + (PULL, "grandchild_2"), ), ), next=("grandchild_2",), @@ -9476,7 +9490,13 @@ def test_doubly_nested_graph_state( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="parent_2"),), + tasks=( + PregelTask( + id=AnyStr(), + name="parent_2", + path=(PULL, "parent_2"), + ), + ), ), StateSnapshot( values={"my_key": "hi my value"}, @@ -9484,6 +9504,7 @@ def test_doubly_nested_graph_state( PregelTask( AnyStr(), "child", + (PULL, "child"), state={ "configurable": { "thread_id": "1", @@ -9534,7 +9555,7 @@ def test_doubly_nested_graph_state( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="parent_1"),), + tasks=(PregelTask(id=AnyStr(), name="parent_1", path=(PULL, "parent_1")),), ), StateSnapshot( values={}, @@ -9554,7 +9575,9 @@ def test_doubly_nested_graph_state( }, created_at=AnyStr(), parent_config=None, - tasks=(PregelTask(id=AnyStr(), name="__start__"),), + tasks=( + PregelTask(id=AnyStr(), name="__start__", path=(PULL, "__start__")), + ), ), ] # get child graph history @@ -9620,6 +9643,7 @@ def test_doubly_nested_graph_state( PregelTask( id=AnyStr(), name="child_1", + path=(PULL, "child_1"), state={ "configurable": { "thread_id": "1", @@ -9650,7 +9674,9 @@ def test_doubly_nested_graph_state( }, created_at=AnyStr(), parent_config=None, - tasks=(PregelTask(id=AnyStr(), name="__start__"),), + tasks=( + PregelTask(id=AnyStr(), name="__start__", path=(PULL, "__start__")), + ), ), ] # get grandchild graph history @@ -9730,7 +9756,11 @@ def test_doubly_nested_graph_state( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="grandchild_2"),), + tasks=( + PregelTask( + id=AnyStr(), name="grandchild_2", path=(PULL, "grandchild_2") + ), + ), ), StateSnapshot( values={"my_key": "hi my value"}, @@ -9768,7 +9798,11 @@ def test_doubly_nested_graph_state( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="grandchild_1"),), + tasks=( + PregelTask( + id=AnyStr(), name="grandchild_1", path=(PULL, "grandchild_1") + ), + ), ), StateSnapshot( values={}, @@ -9800,7 +9834,9 @@ def test_doubly_nested_graph_state( }, created_at=AnyStr(), parent_config=None, - tasks=(PregelTask(id=AnyStr(), name="__start__"),), + tasks=( + PregelTask(id=AnyStr(), name="__start__", path=(PULL, "__start__")), + ), ), ] @@ -9875,6 +9911,7 @@ def test_send_to_nested_graphs( PregelTask( AnyStr(), "generate_joke", + (PUSH, 0), state={ "configurable": { "thread_id": "1", @@ -9885,6 +9922,7 @@ def test_send_to_nested_graphs( PregelTask( AnyStr(), "generate_joke", + (PUSH, 1), state={ "configurable": { "thread_id": "1", @@ -9942,7 +9980,7 @@ def test_send_to_nested_graphs( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(""), name="generate"),), + tasks=(PregelTask(id=AnyStr(""), name="generate", path=(PULL, "generate")),), ) assert graph.get_state(outer_state.tasks[1].state) == StateSnapshot( values={"subject": "dogs - hohoho", "jokes": []}, @@ -9974,7 +10012,7 @@ def test_send_to_nested_graphs( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(""), name="generate"),), + tasks=(PregelTask(id=AnyStr(""), name="generate", path=(PULL, "generate")),), ) # update state of dogs joke graph graph.update_state(outer_state.tasks[1].state, {"subject": "turtles - hohoho"}) @@ -10069,6 +10107,7 @@ def test_send_to_nested_graphs( PregelTask( AnyStr(), "generate_joke", + (PUSH, 0), state={ "configurable": { "thread_id": "1", @@ -10079,6 +10118,7 @@ def test_send_to_nested_graphs( PregelTask( AnyStr(), "generate_joke", + (PUSH, 1), state={ "configurable": { "thread_id": "1", @@ -10107,7 +10147,7 @@ def test_send_to_nested_graphs( ), StateSnapshot( values={"jokes": []}, - tasks=(PregelTask(AnyStr(), "__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), next=("__start__",), config={ "configurable": { @@ -10284,6 +10324,7 @@ def test_weather_subgraph( PregelTask( id=AnyStr(), name="weather_graph", + path=(PULL, "weather_graph"), state={ "configurable": { "thread_id": "1", @@ -10369,6 +10410,7 @@ def test_weather_subgraph( PregelTask( id=AnyStr(), name="weather_graph", + path=(PULL, "weather_graph"), state=StateSnapshot( values={ "messages": [ @@ -10404,7 +10446,13 @@ def test_weather_subgraph( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="weather_node"),), + tasks=( + PregelTask( + id=AnyStr(), + name="weather_node", + path=(PULL, "weather_node"), + ), + ), ), ), ), @@ -10446,6 +10494,7 @@ def test_weather_subgraph( PregelTask( id=AnyStr(), name="weather_graph", + path=(PULL, "weather_graph"), state=StateSnapshot( values={ "messages": [ diff --git a/libs/langgraph/tests/test_pregel_async.py b/libs/langgraph/tests/test_pregel_async.py index 7697610cf..1c1be830e 100644 --- a/libs/langgraph/tests/test_pregel_async.py +++ b/libs/langgraph/tests/test_pregel_async.py @@ -50,7 +50,7 @@ from langgraph.checkpoint.base import ( CheckpointTuple, ) from langgraph.checkpoint.memory import MemorySaver -from langgraph.constants import ERROR, Interrupt, Send +from langgraph.constants import ERROR, PULL, PUSH, Interrupt, Send from langgraph.errors import InvalidUpdateError, NodeInterrupt from langgraph.graph import END, Graph, StateGraph from langgraph.graph.graph import START @@ -298,6 +298,7 @@ async def test_dynamic_interrupt(checkpointer_name: str) -> None: PregelTask( AnyStr(), "tool_two", + (PULL, "tool_two"), interrupts=(Interrupt("Just because..."),), ), ), @@ -927,7 +928,7 @@ async def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 4, "output": 4, "input": 3}, - tasks=(PregelTask(AnyStr(), "two"),), + tasks=(PregelTask(AnyStr(), "two", (PULL, "two")),), next=("two",), config={ "configurable": { @@ -947,7 +948,7 @@ async def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 21, "output": 4, "input": 3}, - tasks=(PregelTask(AnyStr(), "one"),), + tasks=(PregelTask(AnyStr(), "one", (PULL, "one")),), next=("one",), config={ "configurable": { @@ -967,7 +968,7 @@ async def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 21, "output": 4, "input": 20}, - tasks=(PregelTask(AnyStr(), "two"),), + tasks=(PregelTask(AnyStr(), "two", (PULL, "two")),), next=("two",), config={ "configurable": { @@ -987,7 +988,7 @@ async def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 3, "output": 4, "input": 20}, - tasks=(PregelTask(AnyStr(), "one"),), + tasks=(PregelTask(AnyStr(), "one", (PULL, "one")),), next=("one",), config={ "configurable": { @@ -1027,7 +1028,7 @@ async def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"inbox": 3, "input": 2}, - tasks=(PregelTask(AnyStr(), "two"),), + tasks=(PregelTask(AnyStr(), "two", (PULL, "two")),), next=("two",), config={ "configurable": { @@ -1047,7 +1048,7 @@ async def test_invoke_two_processes_in_out_interrupt( ), StateSnapshot( values={"input": 2}, - tasks=(PregelTask(AnyStr(), "one"),), + tasks=(PregelTask(AnyStr(), "one", (PULL, "one")),), next=("one",), config={ "configurable": { @@ -1141,7 +1142,7 @@ async def test_fork_always_re_runs_nodes( ), StateSnapshot( values=5, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -1161,7 +1162,7 @@ async def test_fork_always_re_runs_nodes( ), StateSnapshot( values=4, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -1181,7 +1182,7 @@ async def test_fork_always_re_runs_nodes( ), StateSnapshot( values=3, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -1201,7 +1202,7 @@ async def test_fork_always_re_runs_nodes( ), StateSnapshot( values=2, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -1221,7 +1222,7 @@ async def test_fork_always_re_runs_nodes( ), StateSnapshot( values=1, - tasks=(PregelTask(AnyStr(), "add_one"),), + tasks=(PregelTask(AnyStr(), "add_one", (PULL, "add_one")),), next=("add_one",), config={ "configurable": { @@ -1236,7 +1237,7 @@ async def test_fork_always_re_runs_nodes( ), StateSnapshot( values=0, - tasks=(PregelTask(AnyStr(), "__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), next=("__start__",), config={ "configurable": { @@ -1659,8 +1660,13 @@ async def test_pending_writes_resume( assert state.values == {"value": 1} assert state.next == ("one", "two") assert state.tasks == ( - PregelTask(AnyStr(), "one"), - PregelTask(AnyStr(), "two", 'ConnectionError("I\'m not good")'), + PregelTask(AnyStr(), "one", (PULL, "one")), + PregelTask( + AnyStr(), + "two", + (PULL, "two"), + 'ConnectionError("I\'m not good")', + ), ) assert state.metadata == { "parents": {}, @@ -2524,7 +2530,7 @@ async def test_conditional_graph(checkpointer_name: str) -> None: ), }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -2575,7 +2581,7 @@ async def test_conditional_graph(checkpointer_name: str) -> None: "input": "what is weather in sf", }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -2763,7 +2769,7 @@ async def test_conditional_graph(checkpointer_name: str) -> None: ), }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -2814,7 +2820,7 @@ async def test_conditional_graph(checkpointer_name: str) -> None: "input": "what is weather in sf", }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -3002,7 +3008,7 @@ async def test_conditional_graph(checkpointer_name: str) -> None: ), }, }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -3427,7 +3433,7 @@ async def test_conditional_graph_state( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -3473,7 +3479,7 @@ async def test_conditional_graph_state( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -3610,7 +3616,7 @@ async def test_conditional_graph_state( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -3655,7 +3661,7 @@ async def test_conditional_graph_state( ), "intermediate_steps": [], }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -4297,7 +4303,7 @@ async def test_state_graph_packets(checkpointer_name: str) -> None: ), ] }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PUSH, 0)),), next=("tools",), config=(await app_w_interrupt.checkpointer.aget_tuple(config)).config, created_at=( @@ -4352,7 +4358,7 @@ async def test_state_graph_packets(checkpointer_name: str) -> None: ), ] }, - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PUSH, 0)),), next=("tools",), config=tup.config, created_at=tup.checkpoint["ts"], @@ -4452,7 +4458,10 @@ async def test_state_graph_packets(checkpointer_name: str) -> None: ), ] }, - tasks=(PregelTask(AnyStr(), "tools"), PregelTask(AnyStr(), "tools")), + tasks=( + PregelTask(AnyStr(), "tools", (PUSH, 0)), + PregelTask(AnyStr(), "tools", (PUSH, 1)), + ), next=("tools", "tools"), config=tup.config, created_at=tup.checkpoint["ts"], @@ -4767,7 +4776,7 @@ async def test_message_graph(checkpointer_name: str) -> None: id="ai1", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=tup.config, created_at=tup.checkpoint["ts"], @@ -4816,7 +4825,7 @@ async def test_message_graph(checkpointer_name: str) -> None: ], ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=tup.config, created_at=tup.checkpoint["ts"], @@ -4900,7 +4909,7 @@ async def test_message_graph(checkpointer_name: str) -> None: id="ai2", ), ], - tasks=(PregelTask(AnyStr(), "tools"),), + tasks=(PregelTask(AnyStr(), "tools", (PULL, "tools")),), next=("tools",), config=tup.config, created_at=tup.checkpoint["ts"], @@ -5276,7 +5285,7 @@ async def test_start_branch_then(checkpointer_name: str) -> None: ] assert await tool_two.aget_state(thread1) == StateSnapshot( values={"my_key": "value", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "tool_two_slow"),), + tasks=(PregelTask(AnyStr(), "tool_two_slow", (PULL, "tool_two_slow")),), next=("tool_two_slow",), config=(await tool_two.checkpointer.aget_tuple(thread1)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread1)).checkpoint[ @@ -5319,7 +5328,7 @@ async def test_start_branch_then(checkpointer_name: str) -> None: } assert await tool_two.aget_state(thread2) == StateSnapshot( values={"my_key": "value", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=(await tool_two.checkpointer.aget_tuple(thread2)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread2)).checkpoint[ @@ -5362,7 +5371,7 @@ async def test_start_branch_then(checkpointer_name: str) -> None: } assert await tool_two.aget_state(thread3) == StateSnapshot( values={"my_key": "value", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=(await tool_two.checkpointer.aget_tuple(thread3)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread3)).checkpoint[ @@ -5377,7 +5386,7 @@ async def test_start_branch_then(checkpointer_name: str) -> None: await tool_two.aupdate_state(thread3, {"my_key": "key"}) # appends to my_key assert await tool_two.aget_state(thread3) == StateSnapshot( values={"my_key": "valuekey", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=(await tool_two.checkpointer.aget_tuple(thread3)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread3)).checkpoint[ @@ -5809,7 +5818,7 @@ async def test_branch_then(checkpointer_name: str) -> None: ] assert await tool_two.aget_state(thread1) == StateSnapshot( values={"my_key": "value prepared", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "tool_two_slow"),), + tasks=(PregelTask(AnyStr(), "tool_two_slow", (PULL, "tool_two_slow")),), next=("tool_two_slow",), config=(await tool_two.checkpointer.aget_tuple(thread1)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread1)).checkpoint[ @@ -5857,7 +5866,7 @@ async def test_branch_then(checkpointer_name: str) -> None: } assert await tool_two.aget_state(thread2) == StateSnapshot( values={"my_key": "value prepared", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=(await tool_two.checkpointer.aget_tuple(thread2)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread2)).checkpoint[ @@ -5913,7 +5922,7 @@ async def test_branch_then(checkpointer_name: str) -> None: } assert await tool_two.aget_state(thread1) == StateSnapshot( values={"my_key": "value prepared", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "tool_two_slow"),), + tasks=(PregelTask(AnyStr(), "tool_two_slow", (PULL, "tool_two_slow")),), next=("tool_two_slow",), config=(await tool_two.checkpointer.aget_tuple(thread1)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread1)).checkpoint[ @@ -5961,7 +5970,7 @@ async def test_branch_then(checkpointer_name: str) -> None: } assert await tool_two.aget_state(thread2) == StateSnapshot( values={"my_key": "value prepared", "market": "US"}, - tasks=(PregelTask(AnyStr(), "tool_two_fast"),), + tasks=(PregelTask(AnyStr(), "tool_two_fast", (PULL, "tool_two_fast")),), next=("tool_two_fast",), config=(await tool_two.checkpointer.aget_tuple(thread2)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread2)).checkpoint[ @@ -6009,7 +6018,7 @@ async def test_branch_then(checkpointer_name: str) -> None: # check current state assert await tool_two.aget_state(thread3) == StateSnapshot( values={"my_key": "key", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "prepare"),), + tasks=(PregelTask(AnyStr(), "prepare", (PULL, "prepare")),), next=("prepare",), config=uconfig, created_at=AnyStr(), @@ -6029,7 +6038,7 @@ async def test_branch_then(checkpointer_name: str) -> None: # get state after first node assert await tool_two.aget_state(thread3) == StateSnapshot( values={"my_key": "key prepared", "market": "DE"}, - tasks=(PregelTask(AnyStr(), "tool_two_slow"),), + tasks=(PregelTask(AnyStr(), "tool_two_slow", (PULL, "tool_two_slow")),), next=("tool_two_slow",), config=(await tool_two.checkpointer.aget_tuple(thread3)).config, created_at=(await tool_two.checkpointer.aget_tuple(thread3)).checkpoint[ @@ -7236,6 +7245,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "inner", + (PULL, "inner"), state={ "configurable": {"thread_id": "1", "checkpoint_ns": AnyStr()} }, @@ -7271,6 +7281,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "inner", + (PULL, "inner"), state=StateSnapshot( values={ "my_key": "hi my value here", @@ -7280,6 +7291,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), name="inner_2", + path=(PULL, "inner_2"), error=None, ), ), @@ -7350,6 +7362,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "inner", + (PULL, "inner"), state={ "configurable": { "thread_id": "1", @@ -7383,7 +7396,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: ), StateSnapshot( values={"my_key": "my value"}, - tasks=(PregelTask(AnyStr(), "outer_1"),), + tasks=(PregelTask(AnyStr(), "outer_1", (PULL, "outer_1")),), next=("outer_1",), config={ "configurable": { @@ -7409,7 +7422,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: ), StateSnapshot( values={}, - tasks=(PregelTask(AnyStr(), "__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), next=("__start__",), config={ "configurable": { @@ -7465,7 +7478,9 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="inner_2"),), + tasks=( + PregelTask(id=AnyStr(), name="inner_2", path=(PULL, "inner_2")), + ), ), StateSnapshot( values={"my_key": "hi my value"}, @@ -7494,7 +7509,9 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="inner_1"),), + tasks=( + PregelTask(id=AnyStr(), name="inner_1", path=(PULL, "inner_1")), + ), ), StateSnapshot( values={}, @@ -7517,7 +7534,9 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: }, created_at=AnyStr(), parent_config=None, - tasks=(PregelTask(id=AnyStr(), name="__start__"),), + tasks=( + PregelTask(id=AnyStr(), name="__start__", path=(PULL, "__start__")), + ), ), ] @@ -7587,7 +7606,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: ), StateSnapshot( values={"my_key": "hi my value here and there"}, - tasks=(PregelTask(AnyStr(), "outer_2"),), + tasks=(PregelTask(AnyStr(), "outer_2", (PULL, "outer_2")),), next=("outer_2",), config={ "configurable": { @@ -7617,6 +7636,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "inner", + (PULL, "inner"), state={ "configurable": { "thread_id": "1", @@ -7650,7 +7670,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: ), StateSnapshot( values={"my_key": "my value"}, - tasks=(PregelTask(AnyStr(), "outer_1"),), + tasks=(PregelTask(AnyStr(), "outer_1", (PULL, "outer_1")),), next=("outer_1",), config={ "configurable": { @@ -7676,7 +7696,7 @@ async def test_nested_graph_state(checkpointer_name: str) -> None: ), StateSnapshot( values={}, - tasks=(PregelTask(AnyStr(), "__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), next=("__start__",), config={ "configurable": { @@ -7772,6 +7792,7 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "child", + (PULL, "child"), state={ "configurable": { "thread_id": "1", @@ -7812,6 +7833,7 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "child_1", + (PULL, "child_1"), state={ "configurable": { "thread_id": "1", @@ -7854,6 +7876,7 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "grandchild_2", + (PULL, "grandchild_2"), ), ), next=("grandchild_2",), @@ -7898,18 +7921,21 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "child", + (PULL, "child"), state=StateSnapshot( values={"my_key": "hi my value"}, tasks=( PregelTask( AnyStr(), "child_1", + (PULL, "child_1"), state=StateSnapshot( values={"my_key": "hi my value here"}, tasks=( PregelTask( AnyStr(), "grandchild_2", + (PULL, "grandchild_2"), ), ), next=("grandchild_2",), @@ -8113,7 +8139,11 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="parent_2"),), + tasks=( + PregelTask( + id=AnyStr(), name="parent_2", path=(PULL, "parent_2") + ), + ), ), StateSnapshot( values={"my_key": "hi my value"}, @@ -8121,6 +8151,7 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: PregelTask( AnyStr(), "child", + (PULL, "child"), state={ "configurable": { "thread_id": "1", @@ -8176,7 +8207,11 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="parent_1"),), + tasks=( + PregelTask( + id=AnyStr(), name="parent_1", path=(PULL, "parent_1") + ), + ), ), StateSnapshot( values={}, @@ -8196,7 +8231,11 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: }, created_at=AnyStr(), parent_config=None, - tasks=(PregelTask(id=AnyStr(), name="__start__"),), + tasks=( + PregelTask( + id=AnyStr(), name="__start__", path=(PULL, "__start__") + ), + ), ), ][0] ) @@ -8265,6 +8304,7 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: PregelTask( id=AnyStr(), name="child_1", + path=(PULL, "child_1"), state={ "configurable": { "thread_id": "1", @@ -8295,7 +8335,9 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: }, created_at=AnyStr(), parent_config=None, - tasks=(PregelTask(id=AnyStr(), name="__start__"),), + tasks=( + PregelTask(id=AnyStr(), name="__start__", path=(PULL, "__start__")), + ), ), ] # get grandchild graph history @@ -8379,7 +8421,11 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="grandchild_2"),), + tasks=( + PregelTask( + id=AnyStr(), name="grandchild_2", path=(PULL, "grandchild_2") + ), + ), ), StateSnapshot( values={"my_key": "hi my value"}, @@ -8417,7 +8463,11 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="grandchild_1"),), + tasks=( + PregelTask( + id=AnyStr(), name="grandchild_1", path=(PULL, "grandchild_1") + ), + ), ), StateSnapshot( values={}, @@ -8449,7 +8499,9 @@ async def test_doubly_nested_graph_state(checkpointer_name: str) -> None: }, created_at=AnyStr(), parent_config=None, - tasks=(PregelTask(id=AnyStr(), name="__start__"),), + tasks=( + PregelTask(id=AnyStr(), name="__start__", path=(PULL, "__start__")), + ), ), ] @@ -8524,6 +8576,7 @@ async def test_send_to_nested_graphs(checkpointer_name: str) -> None: PregelTask( AnyStr(), "generate_joke", + (PUSH, 0), state={ "configurable": { "thread_id": "1", @@ -8534,6 +8587,7 @@ async def test_send_to_nested_graphs(checkpointer_name: str) -> None: PregelTask( AnyStr(), "generate_joke", + (PUSH, 1), state={ "configurable": { "thread_id": "1", @@ -8656,6 +8710,7 @@ async def test_send_to_nested_graphs(checkpointer_name: str) -> None: PregelTask( AnyStr(), "generate_joke", + (PUSH, 0), state={ "configurable": { "thread_id": "1", @@ -8666,6 +8721,7 @@ async def test_send_to_nested_graphs(checkpointer_name: str) -> None: PregelTask( AnyStr(), "generate_joke", + (PUSH, 1), state={ "configurable": { "thread_id": "1", @@ -8693,7 +8749,7 @@ async def test_send_to_nested_graphs(checkpointer_name: str) -> None: ), StateSnapshot( values={"jokes": []}, - tasks=(PregelTask(AnyStr(), "__start__"),), + tasks=(PregelTask(AnyStr(), "__start__", (PULL, "__start__")),), next=("__start__",), config={ "configurable": { @@ -8879,6 +8935,7 @@ async def test_weather_subgraph( PregelTask( id=AnyStr(), name="weather_graph", + path=(PULL, "weather_graph"), state={ "configurable": { "thread_id": "1", @@ -8968,6 +9025,7 @@ async def test_weather_subgraph( PregelTask( id=AnyStr(), name="weather_graph", + path=(PULL, "weather_graph"), state=StateSnapshot( values={ "messages": [ @@ -9003,7 +9061,13 @@ async def test_weather_subgraph( "checkpoint_id": AnyStr(), } }, - tasks=(PregelTask(id=AnyStr(), name="weather_node"),), + tasks=( + PregelTask( + id=AnyStr(), + name="weather_node", + path=(PULL, "weather_node"), + ), + ), ), ), ), @@ -9045,6 +9109,7 @@ async def test_weather_subgraph( PregelTask( id=AnyStr(), name="weather_graph", + path=(PULL, "weather_graph"), state=StateSnapshot( values={ "messages": [ diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index e9fe4291b..5b06180e8 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -118,9 +118,8 @@ class KafkaExecutor(AbstractAsyncContextManager): channels=channels, managed=managed, config=msg["config"], - step=msg["task"]["step"], + step=saved.metadata["step"] + 1, for_execution=True, - is_resuming=msg["task"]["resuming"], ): # execute task, saving writes runner = PregelRunner( diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index cb64f8f25..efa18d9e9 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -8,7 +8,7 @@ from langchain_core.runnables import ensure_config import langgraph.scheduler.kafka.serde as serde from langgraph.constants import CONFIG_KEY_DEDUPE_TASKS, INTERRUPT, SCHEDULED from langgraph.pregel import Pregel -from langgraph.pregel.loop import INPUT_RESUMING, AsyncPregelLoop +from langgraph.pregel.loop import AsyncPregelLoop from langgraph.pregel.types import RetryPolicy from langgraph.scheduler.kafka.retry import aretry from langgraph.scheduler.kafka.types import ( @@ -45,7 +45,6 @@ class KafkaOrchestrator(AbstractAsyncContextManager): self.consumer = await self.stack.enter_async_context( aiokafka.AIOKafkaConsumer( self.topics.orchestrator, - value_deserializer=serde.loads, auto_offset_reset="earliest", group_id=self.group_id, enable_auto_commit=False, @@ -69,9 +68,9 @@ class KafkaOrchestrator(AbstractAsyncContextManager): recs = await self.consumer.getmany( timeout_ms=self.batch_max_ms, max_records=self.batch_max_n ) - msgs: list[MessageToOrchestrator] = [ - msg.value for msgs in recs.values() for msg in msgs - ] + # dedupe messages, eg. if multiple nodes finish around same time + uniq = set(msg.value for msgs in recs.values() for msg in msgs) + msgs: list[MessageToOrchestrator] = [serde.loads(msg) for msg in uniq] except aiokafka.ConsumerStoppedError: raise StopAsyncIteration from None # process batch @@ -130,12 +129,7 @@ class KafkaOrchestrator(AbstractAsyncContextManager): CONFIG_KEY_DEDUPE_TASKS: True, }, ), - task=ExecutorTask( - id=task.id, - path=task.path, - step=loop.step, - resuming=loop.input is INPUT_RESUMING, - ), + task=ExecutorTask(id=task.id, path=task.path), ), ) for task in new_tasks diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py index 78396c88e..a63057319 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py @@ -17,8 +17,6 @@ class MessageToOrchestrator(TypedDict): class ExecutorTask(TypedDict): id: str path: tuple[str, ...] - step: int - resuming: bool class MessageToExecutor(TypedDict): From ad7dd0dbb7cf2ed23ea015d84659dbb7382141cd Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Mon, 9 Sep 2024 16:17:37 -0700 Subject: [PATCH 16/34] Add drain_topics test helper --- libs/langgraph/langgraph/pregel/loop.py | 9 +- libs/scheduler-kafka/tests/any.py | 37 ++++ libs/scheduler-kafka/tests/conftest.py | 6 +- libs/scheduler-kafka/tests/run.py | 97 ++++++++++ libs/scheduler-kafka/tests/test_fanout.py | 217 ++++++++++++---------- 5 files changed, 264 insertions(+), 102 deletions(-) create mode 100644 libs/scheduler-kafka/tests/any.py create mode 100644 libs/scheduler-kafka/tests/run.py diff --git a/libs/langgraph/langgraph/pregel/loop.py b/libs/langgraph/langgraph/pregel/loop.py index 7772827c4..7d4df7092 100644 --- a/libs/langgraph/langgraph/pregel/loop.py +++ b/libs/langgraph/langgraph/pregel/loop.py @@ -336,6 +336,13 @@ class PregelLoop: self.status = "done" return False + print( + self.step, + self.skip_done_tasks, + [(t.id, t.name) for t in self.tasks.values()], + self.checkpoint_pending_writes, + ) + # if there are pending writes from a previous loop, apply them if self.skip_done_tasks and self.checkpoint_pending_writes: for tid, k, v in self.checkpoint_pending_writes: @@ -343,7 +350,7 @@ class PregelLoop: continue if task := self.tasks.get(tid): if k == SCHEDULED: - if v == max( + if True or v == max( self.checkpoint["versions_seen"] .get(INTERRUPT, {}) .values(), diff --git a/libs/scheduler-kafka/tests/any.py b/libs/scheduler-kafka/tests/any.py new file mode 100644 index 000000000..73744a1e8 --- /dev/null +++ b/libs/scheduler-kafka/tests/any.py @@ -0,0 +1,37 @@ +import re +from typing import Union + + +class AnyStr(str): + def __init__(self, prefix: Union[str, re.Pattern] = "") -> None: + super().__init__() + self.prefix = prefix + + def __eq__(self, other: object) -> bool: + return isinstance(other, str) and ( + other.startswith(self.prefix) + if isinstance(self.prefix, str) + else self.prefix.match(other) + ) + + def __hash__(self) -> int: + return hash((str(self), self.prefix)) + + +class AnyDict(dict): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def __eq__(self, other: object) -> bool: + if not self and isinstance(other, dict): + return True + if not isinstance(other, dict) or len(self) != len(other): + return False + for k, v in self.items(): + if kk := next((kk for kk in other if kk == k), None): + if v == other[kk]: + continue + else: + return False + else: + return True diff --git a/libs/scheduler-kafka/tests/conftest.py b/libs/scheduler-kafka/tests/conftest.py index 7d855a113..74b8a7800 100644 --- a/libs/scheduler-kafka/tests/conftest.py +++ b/libs/scheduler-kafka/tests/conftest.py @@ -19,9 +19,9 @@ def anyio_backend(): @pytest.fixture def topics() -> Iterator[Topics]: - o = f"test_{uuid4().hex[:16]}" - e = f"test_{uuid4().hex[:16]}" - z = f"test_{uuid4().hex[:16]}" + o = f"test_o_{uuid4().hex[:16]}" + e = f"test_e_{uuid4().hex[:16]}" + z = f"test_z_{uuid4().hex[:16]}" admin = kafka.admin.KafkaAdminClient() # create topics admin.create_topics( diff --git a/libs/scheduler-kafka/tests/run.py b/libs/scheduler-kafka/tests/run.py new file mode 100644 index 000000000..03b3108c4 --- /dev/null +++ b/libs/scheduler-kafka/tests/run.py @@ -0,0 +1,97 @@ +import asyncio +import functools +from typing import Callable, Optional, ParamSpec, TypeVar + +import anyio +from aiokafka import AIOKafkaConsumer +from langchain_core.runnables import RunnableConfig + +from langgraph.pregel import Pregel +from langgraph.pregel.types import StateSnapshot +from langgraph.scheduler.kafka.executor import KafkaExecutor +from langgraph.scheduler.kafka.orchestrator import KafkaOrchestrator +from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics + +C = ParamSpec("C") +R = TypeVar("R") + + +def timeout(delay: int): + def decorator(func: Callable[C, R]) -> Callable[C, R]: + @functools.wraps(func) + async def new_func(*args: C.args, **kwargs: C.kwargs) -> R: + async with asyncio.timeout(delay): + return await func(*args, **kwargs) + + return new_func + + return decorator + + +@timeout(20) +async def drain_topics( + topics: Topics, + graph: Pregel, + config: RunnableConfig, + *, + until: Callable[[StateSnapshot], bool], + debug: bool = False, +) -> tuple[list[MessageToOrchestrator], list[MessageToOrchestrator]]: + scope: Optional[anyio.CancelScope] = None + orch_msgs = [] + exec_msgs = [] + errors = [] + + async def orchestrator() -> None: + async with KafkaOrchestrator(graph, topics) as orch: + async for msgs in orch: + orch_msgs.extend(msgs) + if debug: + print("orch", len(msgs)) + + async def executor() -> None: + async with KafkaExecutor(graph, topics) as exec: + async for msgs in exec: + exec_msgs.extend(msgs) + if debug: + print("exec", len(msgs)) + + async def error_consumer() -> None: + async with AIOKafkaConsumer(topics.error) as consumer: + async for msg in consumer: + errors.append(msg) + if scope: + scope.cancel() + + async def poller(expected_next: tuple[str, ...]) -> None: + while True: + await asyncio.sleep(0.5) + state = await graph.aget_state(config) + if until(state): + break + if scope: + scope.cancel() + + # start error consumer and poller + error_task = asyncio.create_task(error_consumer(), name="error_consumer") + poller_task = asyncio.create_task(poller(()), name="poller") + + # run the orchestrator and executor until break_when + async with anyio.create_task_group() as tg: + scope = tg.cancel_scope + tg.start_soon(orchestrator, name="orchestrator") + tg.start_soon(executor, name="executor") + + # cancel error consumer and poller + error_task.cancel() + poller_task.cancel() + + try: + await asyncio.gather(error_task, poller_task) + except asyncio.CancelledError: + pass + + # check no errors + assert not errors + + return orch_msgs, exec_msgs diff --git a/libs/scheduler-kafka/tests/test_fanout.py b/libs/scheduler-kafka/tests/test_fanout.py index ac8889045..e00787aee 100644 --- a/libs/scheduler-kafka/tests/test_fanout.py +++ b/libs/scheduler-kafka/tests/test_fanout.py @@ -1,35 +1,24 @@ import asyncio -import functools import operator -from typing import Annotated, Callable, ParamSpec, Sequence, TypedDict, TypeVar, Union +from typing import ( + Annotated, + Sequence, + TypedDict, + Union, +) -import anyio import pytest -from aiokafka import AIOKafkaConsumer, AIOKafkaProducer +from aiokafka import AIOKafkaProducer from langgraph.checkpoint.base import BaseCheckpointSaver from langgraph.graph.state import StateGraph from langgraph.pregel import Pregel from langgraph.scheduler.kafka import serde -from langgraph.scheduler.kafka.executor import KafkaExecutor -from langgraph.scheduler.kafka.orchestrator import KafkaOrchestrator from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics +from tests.any import AnyDict +from tests.run import drain_topics pytestmark = pytest.mark.anyio -C = ParamSpec("C") -R = TypeVar("R") - - -def timeout(delay: int): - def decorator(func: Callable[C, R]) -> Callable[C, R]: - @functools.wraps(func) - async def new_func(*args: C.args, **kwargs: C.kwargs) -> R: - async with asyncio.timeout(delay): - return await func(*args, **kwargs) - - return new_func - - return decorator def mk_fanout_graph( @@ -97,31 +86,10 @@ def mk_fanout_graph( return builder.compile(checkpointer, interrupt_before=interrupt_before) -@timeout(10) async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) -> None: input = {"query": "what is weather in sf"} config = {"configurable": {"thread_id": "1"}} graph = mk_fanout_graph(checkpointer) - n_orch_msgs = 0 - n_exec_msgs = 0 - - async def orchestrator(expected: int) -> None: - nonlocal n_orch_msgs - async with KafkaOrchestrator(graph, topics) as orch: - async for msgs in orch: - print("orch", msgs) - n_orch_msgs += len(msgs) - if n_orch_msgs == expected: - break - - async def executor(expected: int) -> None: - nonlocal n_exec_msgs - async with KafkaExecutor(graph, topics) as exec: - async for msgs in exec: - print("exec", msgs) - n_exec_msgs += len(msgs) - if n_exec_msgs == expected: - break # start a new run async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: @@ -130,20 +98,14 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - MessageToOrchestrator(input=input, config=config), ) - # run the orchestrator and executor - async with anyio.create_task_group() as tg: - tg.start_soon(orchestrator, 13, name="orchestrator") - tg.start_soon(executor, 12, name="executor") - - # check no errors - async with AIOKafkaConsumer(topics.error) as consumer: - assert len(consumer.assignment()) > 0 - for tp in consumer.assignment(): - assert await consumer.position(tp) == 0 + # drain topics + orch_msgs, exec_msgs = await drain_topics( + topics, graph, config, until=lambda s: s.values and s.next == () + ) + # check state state = await graph.aget_state(config) - assert n_orch_msgs == 13 - assert n_exec_msgs == 12 + assert state.next == () assert ( state.values == await graph.ainvoke(input, {"configurable": {"thread_id": "2"}}) @@ -154,34 +116,62 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - } ) + # check history + history = [c async for c in graph.aget_state_history(config)] + assert len(history) == 11 + + # check messages + assert orch_msgs == [MessageToOrchestrator(input=input, config=config)] + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "input": None, + } + for c in reversed(history) + for _ in c.tasks + ] + assert exec_msgs == [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "task": { + "id": t.id, + "path": list(t.path), + }, + } + for c in reversed(history) + for t in c.tasks + ] + -@timeout(10) async def test_fanout_graph_w_interrupt( topics: Topics, checkpointer: BaseCheckpointSaver ) -> None: input = {"query": "what is weather in sf"} config = {"configurable": {"thread_id": "1"}} graph = mk_fanout_graph(checkpointer, interrupt_before=["qa"]) - n_orch_msgs = 0 - n_exec_msgs = 0 - - async def orchestrator(expected: int) -> None: - nonlocal n_orch_msgs - async with KafkaOrchestrator(graph, topics) as orch: - async for msgs in orch: - print("orch", msgs) - n_orch_msgs += len(msgs) - if n_orch_msgs == expected: - break - - async def executor(expected: int) -> None: - nonlocal n_exec_msgs - async with KafkaExecutor(graph, topics) as exec: - async for msgs in exec: - print("exec", msgs) - n_exec_msgs += len(msgs) - if n_exec_msgs == expected: - break # start a new run async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: @@ -190,21 +180,12 @@ async def test_fanout_graph_w_interrupt( MessageToOrchestrator(input=input, config=config), ) - # run the orchestrator and executor - async with anyio.create_task_group() as tg: - tg.start_soon(orchestrator, 12, name="orchestrator") - tg.start_soon(executor, 11, name="executor") - - # check no errors - async with AIOKafkaConsumer(topics.error) as consumer: - assert len(consumer.assignment()) > 0 - for tp in consumer.assignment(): - assert await consumer.position(tp) == 0 + orch_msgs, exec_msgs = await drain_topics( + topics, graph, config, until=lambda s: s.values and s.next == ("qa",) + ) # check interrupted state state = await graph.aget_state(config) - assert n_orch_msgs == 12 - assert n_exec_msgs == 11 assert state.next == ("qa",) assert ( state.values @@ -215,6 +196,55 @@ async def test_fanout_graph_w_interrupt( } ) + # check history + history = [c async for c in graph.aget_state_history(config)] + assert len(history) == 10 + + # check messages + assert orch_msgs == [MessageToOrchestrator(input=input, config=config)] + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "input": None, + } + for c in reversed(history[1:]) # the last one wasn't executed + for _ in c.tasks + ] + assert exec_msgs == [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "task": { + "id": t.id, + "path": list(t.path), + }, + } + for c in reversed(history[1:]) # the last one wasn't executed + for t in c.tasks + ] + # resume the thread async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: await producer.send_and_wait( @@ -222,21 +252,12 @@ async def test_fanout_graph_w_interrupt( MessageToOrchestrator(input=None, config=config), ) - # run the orchestrator and executor - async with anyio.create_task_group() as tg: - tg.start_soon(orchestrator, 14, name="orchestrator") - tg.start_soon(executor, 12, name="executor") - - # check no errors - async with AIOKafkaConsumer(topics.error) as consumer: - assert len(consumer.assignment()) > 0 - for tp in consumer.assignment(): - assert await consumer.position(tp) == 0 + orch_msgs, exec_msgs = await drain_topics( + topics, graph, config, until=lambda s: s.values and s.next == () + ) # check final state state = await graph.aget_state(config) - assert n_orch_msgs == 14 - assert n_exec_msgs == 12 assert state.next == () assert ( state.values From 030d6d2def82c03c9d3e7f7eed44d3028476987d Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Mon, 9 Sep 2024 18:07:39 -0700 Subject: [PATCH 17/34] Fix subgraph test --- libs/langgraph/langgraph/pregel/algo.py | 1 + libs/langgraph/langgraph/pregel/loop.py | 9 +- .../langgraph/scheduler/kafka/executor.py | 2 +- libs/scheduler-kafka/tests/run.py | 2 +- libs/scheduler-kafka/tests/test_subgraph.py | 116 ++++-------------- 5 files changed, 26 insertions(+), 104 deletions(-) diff --git a/libs/langgraph/langgraph/pregel/algo.py b/libs/langgraph/langgraph/pregel/algo.py index 2f947be22..d0f8e712e 100644 --- a/libs/langgraph/langgraph/pregel/algo.py +++ b/libs/langgraph/langgraph/pregel/algo.py @@ -518,6 +518,7 @@ def prepare_single_task( **configurable.get(CONFIG_KEY_CHECKPOINT_MAP, {}), parent_ns: checkpoint["id"], }, + "checkpoint_id": None, "checkpoint_ns": task_checkpoint_ns, }, ), diff --git a/libs/langgraph/langgraph/pregel/loop.py b/libs/langgraph/langgraph/pregel/loop.py index 7d4df7092..7772827c4 100644 --- a/libs/langgraph/langgraph/pregel/loop.py +++ b/libs/langgraph/langgraph/pregel/loop.py @@ -336,13 +336,6 @@ class PregelLoop: self.status = "done" return False - print( - self.step, - self.skip_done_tasks, - [(t.id, t.name) for t in self.tasks.values()], - self.checkpoint_pending_writes, - ) - # if there are pending writes from a previous loop, apply them if self.skip_done_tasks and self.checkpoint_pending_writes: for tid, k, v in self.checkpoint_pending_writes: @@ -350,7 +343,7 @@ class PregelLoop: continue if task := self.tasks.get(tid): if k == SCHEDULED: - if True or v == max( + if v == max( self.checkpoint["versions_seen"] .get(INTERRUPT, {}) .values(), diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index 5b06180e8..91b53f31d 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -120,6 +120,7 @@ class KafkaExecutor(AbstractAsyncContextManager): config=msg["config"], step=saved.metadata["step"] + 1, for_execution=True, + checkpointer=self.graph.checkpointer, ): # execute task, saving writes runner = PregelRunner( @@ -146,5 +147,4 @@ class KafkaExecutor(AbstractAsyncContextManager): task_id: str, writes: list[tuple[str, Any]], ) -> None: - print("put_writes", task_id, writes) return submit(self.graph.checkpointer.aput_writes, config, writes, task_id) diff --git a/libs/scheduler-kafka/tests/run.py b/libs/scheduler-kafka/tests/run.py index 03b3108c4..6543dd0ee 100644 --- a/libs/scheduler-kafka/tests/run.py +++ b/libs/scheduler-kafka/tests/run.py @@ -92,6 +92,6 @@ async def drain_topics( pass # check no errors - assert not errors + assert not errors, errors return orch_msgs, exec_msgs diff --git a/libs/scheduler-kafka/tests/test_subgraph.py b/libs/scheduler-kafka/tests/test_subgraph.py index 15bb52ac2..f051d1369 100644 --- a/libs/scheduler-kafka/tests/test_subgraph.py +++ b/libs/scheduler-kafka/tests/test_subgraph.py @@ -1,11 +1,7 @@ -import asyncio -import functools -import re -from typing import Callable, Literal, Optional, ParamSpec, TypeVar, Union, cast +from typing import Literal, ParamSpec, TypeVar, cast -import anyio import pytest -from aiokafka import AIOKafkaConsumer, AIOKafkaProducer +from aiokafka import AIOKafkaProducer from langchain_core.language_models.fake_chat_models import ( FakeMessagesListChatModel, ) @@ -18,43 +14,15 @@ from langgraph.graph import MessagesState from langgraph.graph.state import StateGraph from langgraph.pregel import Pregel from langgraph.scheduler.kafka import serde -from langgraph.scheduler.kafka.executor import KafkaExecutor -from langgraph.scheduler.kafka.orchestrator import KafkaOrchestrator from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics +from tests.any import AnyStr +from tests.run import drain_topics pytestmark = pytest.mark.anyio C = ParamSpec("C") R = TypeVar("R") -class AnyStr(str): - def __init__(self, prefix: Union[str, re.Pattern] = "") -> None: - super().__init__() - self.prefix = prefix - - def __eq__(self, other: object) -> bool: - return isinstance(other, str) and ( - other.startswith(self.prefix) - if isinstance(self.prefix, str) - else self.prefix.match(other) - ) - - def __hash__(self) -> int: - return hash((str(self), self.prefix)) - - -def timeout(delay: int): - def decorator(func: Callable[C, R]) -> Callable[C, R]: - @functools.wraps(func) - async def new_func(*args: C.args, **kwargs: C.kwargs) -> R: - async with asyncio.timeout(delay): - return await func(*args, **kwargs) - - return new_func - - return decorator - - def mk_weather_graph(checkpointer: BaseCheckpointSaver) -> Pregel: # copied from test_weather_subgraph @@ -149,45 +117,12 @@ def mk_weather_graph(checkpointer: BaseCheckpointSaver) -> Pregel: return graph.compile(checkpointer=checkpointer) -@timeout(10) async def test_subgraph_w_interrupt( topics: Topics, checkpointer: BaseCheckpointSaver ) -> None: input = {"messages": [{"role": "user", "content": "what's the weather in sf"}]} config = {"configurable": {"thread_id": "1"}} graph = mk_weather_graph(checkpointer) - n_orch_msgs = 0 - n_exec_msgs = 0 - errors = [] - scope: Optional[anyio.CancelScope] = None - - async def orchestrator(expected: int) -> None: - nonlocal n_orch_msgs - async with KafkaOrchestrator(graph, topics) as orch: - async for msgs in orch: - n_orch_msgs += len(msgs) - print("orch", n_orch_msgs, msgs) - if n_orch_msgs == expected: - break - - async def executor(expected: int) -> None: - nonlocal n_exec_msgs - async with KafkaExecutor(graph, topics) as exec: - async for msgs in exec: - n_exec_msgs += len(msgs) - print("exec", n_exec_msgs, msgs) - if n_exec_msgs == expected: - break - - async def error_consumer() -> None: - async with AIOKafkaConsumer(topics.error) as consumer: - async for msg in consumer: - errors.append(msg) - if scope: - scope.cancel() - - # start error consumer - error_task = asyncio.create_task(error_consumer(), name="error_consumer") # start a new run async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: @@ -196,19 +131,17 @@ async def test_subgraph_w_interrupt( MessageToOrchestrator(input=input, config=config), ) - # run the orchestrator and executor - async with anyio.create_task_group() as tg: - scope = tg.cancel_scope - tg.start_soon(orchestrator, 4, name="orchestrator") - tg.start_soon(executor, 3, name="executor") - - # check no errors - assert not errors + orch_msgs, exec_msgs = await drain_topics( + topics, + graph, + config, + until=lambda state: state.next == ("weather_graph",), + ) # check interrupted state state = await graph.aget_state(config) - assert n_orch_msgs == 4 - assert n_exec_msgs == 3 + assert len(orch_msgs) == 4 + assert len(exec_msgs) == 3 assert state.next == ("weather_graph",) assert state.values == { "messages": [HumanMessage(id=AnyStr(), content="what's the weather in sf")], @@ -222,24 +155,19 @@ async def test_subgraph_w_interrupt( MessageToOrchestrator(input=None, config=config), ) - # run the orchestrator and executor - async with anyio.create_task_group() as tg: - scope = tg.cancel_scope - tg.start_soon(orchestrator, 6, name="orchestrator") - tg.start_soon(executor, 4, name="executor") - - # check no errors - assert not errors + orch_msgs, exec_msgs = await drain_topics( + topics, graph, config, until=lambda state: state.next == (), debug=True + ) # check final state state = await graph.aget_state(config) - assert n_orch_msgs == 6 - assert n_exec_msgs == 4 + assert len(orch_msgs) == 2 + assert len(exec_msgs) == 1 assert state.next == () assert state.values == { - "answer": "doc1,doc1,doc2,doc2,doc3,doc3,doc4,doc4", - "docs": ["doc1", "doc1", "doc2", "doc2", "doc3", "doc3", "doc4", "doc4"], - "query": "analyzed: query: analyzed: query: what is weather in sf", + "messages": [ + HumanMessage(id=AnyStr(), content="what's the weather in sf"), + AIMessage(content="I'ts sunny in San Francisco!", id=AnyStr()), + ], + "route": "weather", } - - error_task.cancel() From 8f914026429fabddd704ae1a8459beb663bd361b Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 12:26:07 -0700 Subject: [PATCH 18/34] Use thread_id for partition key, ignore tasks for stale checkpoints --- .../langgraph/checkpoint/postgres/aio.py | 20 ++++--- libs/langgraph/langgraph/constants.py | 4 ++ libs/langgraph/langgraph/errors.py | 6 +++ libs/langgraph/langgraph/pregel/__init__.py | 5 +- libs/langgraph/langgraph/pregel/algo.py | 9 +++- libs/langgraph/langgraph/pregel/loop.py | 53 +++++++++++++++---- libs/langgraph/langgraph/pregel/runner.py | 15 ++++-- .../langgraph/scheduler/kafka/executor.py | 13 ++++- .../langgraph/scheduler/kafka/orchestrator.py | 11 +++- libs/scheduler-kafka/tests/test_fanout.py | 4 ++ 10 files changed, 107 insertions(+), 33 deletions(-) diff --git a/libs/checkpoint-postgres/langgraph/checkpoint/postgres/aio.py b/libs/checkpoint-postgres/langgraph/checkpoint/postgres/aio.py index 488d39e99..9c246ef74 100644 --- a/libs/checkpoint-postgres/langgraph/checkpoint/postgres/aio.py +++ b/libs/checkpoint-postgres/langgraph/checkpoint/postgres/aio.py @@ -298,18 +298,16 @@ class AsyncPostgresSaver(BasePostgresSaver): if all(w[0] in WRITES_IDX_MAP for w in writes) else self.INSERT_CHECKPOINT_WRITES_SQL ) + params = await asyncio.to_thread( + self._dump_writes, + config["configurable"]["thread_id"], + config["configurable"]["checkpoint_ns"], + config["configurable"]["checkpoint_id"], + task_id, + writes, + ) async with self._cursor(pipeline=True) as cur: - await cur.executemany( - query, - await asyncio.to_thread( - self._dump_writes, - config["configurable"]["thread_id"], - config["configurable"]["checkpoint_ns"], - config["configurable"]["checkpoint_id"], - task_id, - writes, - ), - ) + await cur.executemany(query, params) @asynccontextmanager async def _cursor(self, *, pipeline: bool = False) -> AsyncIterator[AsyncCursor]: diff --git a/libs/langgraph/langgraph/constants.py b/libs/langgraph/langgraph/constants.py index 7cea9de85..15dc47fd9 100644 --- a/libs/langgraph/langgraph/constants.py +++ b/libs/langgraph/langgraph/constants.py @@ -10,10 +10,12 @@ CONFIG_KEY_STORE = "__pregel_store" CONFIG_KEY_RESUMING = "__pregel_resuming" CONFIG_KEY_TASK_ID = "__pregel_task_id" CONFIG_KEY_DEDUPE_TASKS = "__pregel_dedupe_tasks" +CONFIG_KEY_ENSURE_LATEST = "__pregel_ensure_latest" # this one part of public API so more readable CONFIG_KEY_CHECKPOINT_MAP = "checkpoint_map" INTERRUPT = "__interrupt__" ERROR = "__error__" +NO_WRITES = "__no_writes__" SCHEDULED = "__scheduled__" TASKS = "__pregel_tasks" # for backwards compat, this is the original name of PUSH PUSH = "__pregel_push" @@ -23,6 +25,7 @@ RESERVED = { SCHEDULED, INTERRUPT, ERROR, + NO_WRITES, TASKS, PUSH, PULL, @@ -34,6 +37,7 @@ RESERVED = { CONFIG_KEY_RESUMING, CONFIG_KEY_TASK_ID, CONFIG_KEY_DEDUPE_TASKS, + CONFIG_KEY_ENSURE_LATEST, INPUT, RUNTIME_PLACEHOLDER, } diff --git a/libs/langgraph/langgraph/errors.py b/libs/langgraph/langgraph/errors.py index aa5b57857..2fcd64473 100644 --- a/libs/langgraph/langgraph/errors.py +++ b/libs/langgraph/langgraph/errors.py @@ -55,6 +55,12 @@ class TaskNotFound(Exception): pass +class CheckpointNotLatest(Exception): + """Raised when the checkpoint is not the latest version.""" + + pass + + __all__ = [ "GraphRecursionError", "InvalidUpdateError", diff --git a/libs/langgraph/langgraph/pregel/__init__.py b/libs/langgraph/langgraph/pregel/__init__.py index d1229d6e9..c2324f920 100644 --- a/libs/langgraph/langgraph/pregel/__init__.py +++ b/libs/langgraph/langgraph/pregel/__init__.py @@ -21,7 +21,6 @@ from typing import ( from uuid import UUID, uuid5 from langchain_core.globals import get_debug -from langchain_core.load.dump import dumpd from langchain_core.runnables import ( Runnable, RunnableLambda, @@ -1160,7 +1159,7 @@ class Pregel(Runnable[Union[dict[str, Any], Any], Union[dict[str, Any], Any]]): config = ensure_config(merge_configs(self.config, config)) callback_manager = get_callback_manager_for_config(config) run_manager = callback_manager.on_chain_start( - dumpd(self), + None, input, name=config.get("run_name", self.get_name()), run_id=config.get("run_id"), @@ -1341,7 +1340,7 @@ class Pregel(Runnable[Union[dict[str, Any], Any], Union[dict[str, Any], Any]]): config = ensure_config(merge_configs(self.config, config)) callback_manager = get_async_callback_manager_for_config(config) run_manager = await callback_manager.on_chain_start( - dumpd(self), + None, input, name=config.get("run_name", self.get_name()), run_id=config.get("run_id"), diff --git a/libs/langgraph/langgraph/pregel/algo.py b/libs/langgraph/langgraph/pregel/algo.py index d0f8e712e..9f9837980 100644 --- a/libs/langgraph/langgraph/pregel/algo.py +++ b/libs/langgraph/langgraph/pregel/algo.py @@ -28,6 +28,7 @@ from langgraph.constants import ( CONFIG_KEY_SEND, CONFIG_KEY_TASK_ID, INTERRUPT, + NO_WRITES, NS_SEP, PULL, PUSH, @@ -196,7 +197,9 @@ def apply_writes( pending_writes_by_managed: dict[str, list[Any]] = defaultdict(list) for task in tasks: for chan, val in task.writes: - if chan == TASKS: + if chan == NO_WRITES: + pass + elif chan == TASKS: checkpoint["pending_sends"].append(val) elif chan in channels: pending_writes_by_channel[chan].append(val) @@ -331,6 +334,8 @@ def prepare_single_task( if task_path[0] == PUSH: idx = int(task_path[1]) + if idx >= len(checkpoint["pending_sends"]): + return packet = checkpoint["pending_sends"][idx] if not isinstance(packet, Send): logger.warning( @@ -425,6 +430,8 @@ def prepare_single_task( return PregelTask(task_id, packet.node, task_path) elif task_path[0] == PULL: name = str(task_path[1]) + if name not in processes: + return proc = processes[name] version_type = type(next(iter(checkpoint["channel_versions"].values()), None)) null_version = version_type() diff --git a/libs/langgraph/langgraph/pregel/loop.py b/libs/langgraph/langgraph/pregel/loop.py index 7772827c4..c0551c14b 100644 --- a/libs/langgraph/langgraph/pregel/loop.py +++ b/libs/langgraph/langgraph/pregel/loop.py @@ -40,6 +40,7 @@ from langgraph.checkpoint.base import ( from langgraph.constants import ( CONFIG_KEY_CHECKPOINT_MAP, CONFIG_KEY_DEDUPE_TASKS, + CONFIG_KEY_ENSURE_LATEST, CONFIG_KEY_RESUMING, CONFIG_KEY_STREAM, CONFIG_KEY_TASK_ID, @@ -49,7 +50,7 @@ from langgraph.constants import ( SCHEDULED, TAG_HIDDEN, ) -from langgraph.errors import EmptyInputError, GraphInterrupt +from langgraph.errors import CheckpointNotLatest, EmptyInputError, GraphInterrupt from langgraph.managed.base import ( ManagedValueMapping, ManagedValueSpec, @@ -599,11 +600,26 @@ class SyncPregelLoop(PregelLoop, ContextManager): # context manager def __enter__(self) -> Self: - saved = ( - self.checkpointer.get_tuple(self.checkpoint_config) - if self.checkpointer - else None - ) or CheckpointTuple(self.config, empty_checkpoint(), {"step": -2}, None, []) + if self.config.get("configurable", {}).get( + CONFIG_KEY_ENSURE_LATEST + ) and self.checkpoint_config["configurable"].get("checkpoint_id"): + saved = self.checkpointer.get_tuple( + patch_configurable(self.checkpoint_config, {"checkpoint_id": None}) + ) + if ( + saved is None + or saved.checkpoint["id"] + != self.checkpoint_config["configurable"]["checkpoint_id"] + ): + raise CheckpointNotLatest + elif self.checkpointer: + saved = self.checkpointer.get_tuple(self.checkpoint_config) + else: + saved = None + if saved is None: + saved = CheckpointTuple( + self.config, empty_checkpoint(), {"step": -2}, None, [] + ) self.checkpoint_config = { **self.config, **saved.config, @@ -702,11 +718,26 @@ class AsyncPregelLoop(PregelLoop, AsyncContextManager): # context manager async def __aenter__(self) -> Self: - saved = ( - await self.checkpointer.aget_tuple(self.checkpoint_config) - if self.checkpointer - else None - ) or CheckpointTuple(self.config, empty_checkpoint(), {"step": -2}, None, []) + if self.config.get("configurable", {}).get( + CONFIG_KEY_ENSURE_LATEST + ) and self.checkpoint_config["configurable"].get("checkpoint_id"): + saved = await self.checkpointer.aget_tuple( + patch_configurable(self.checkpoint_config, {"checkpoint_id": None}) + ) + if ( + saved is None + or saved.checkpoint["id"] + != self.checkpoint_config["configurable"]["checkpoint_id"] + ): + raise CheckpointNotLatest + elif self.checkpointer: + saved = await self.checkpointer.aget_tuple(self.checkpoint_config) + else: + saved = None + if saved is None: + saved = CheckpointTuple( + self.config, empty_checkpoint(), {"step": -2}, None, [] + ) self.checkpoint_config = { **self.config, **saved.config, diff --git a/libs/langgraph/langgraph/pregel/runner.py b/libs/langgraph/langgraph/pregel/runner.py index 3f42fbd5f..086afa9da 100644 --- a/libs/langgraph/langgraph/pregel/runner.py +++ b/libs/langgraph/langgraph/pregel/runner.py @@ -12,7 +12,7 @@ from typing import ( Union, ) -from langgraph.constants import ERROR, INTERRUPT +from langgraph.constants import ERROR, INTERRUPT, NO_WRITES from langgraph.errors import GraphInterrupt from langgraph.pregel.executor import Submit from langgraph.pregel.retry import arun_with_retry, run_with_retry @@ -69,12 +69,15 @@ class PregelRunner: if exc := _exception(fut): if isinstance(exc, GraphInterrupt): # save interrupt to checkpointer - self.put_writes(task.id, [(INTERRUPT, i) for i in exc.args[0]]) + if interrupts := [(INTERRUPT, i) for i in exc.args[0]]: + self.put_writes(task.id, interrupts) else: # save error to checkpointer self.put_writes(task.id, [(ERROR, exc)]) - else: + if not task.writes: + # add no writes marker + task.writes.append((NO_WRITES, None)) # save task writes to checkpointer self.put_writes(task.id, task.writes) else: @@ -130,11 +133,15 @@ class PregelRunner: if exc := _exception(fut): if isinstance(exc, GraphInterrupt): # save interrupt to checkpointer - self.put_writes(task.id, [(INTERRUPT, i) for i in exc.args[0]]) + if interrupts := [(INTERRUPT, i) for i in exc.args[0]]: + self.put_writes(task.id, interrupts) else: # save error to checkpointer self.put_writes(task.id, [(ERROR, exc)]) else: + if not task.writes: + # add no writes marker + task.writes.append((NO_WRITES, None)) # save task writes to checkpointer self.put_writes(task.id, task.writes) else: diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index 91b53f31d..714fbb647 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -8,7 +8,7 @@ from langchain_core.runnables import RunnableConfig import langgraph.scheduler.kafka.serde as serde from langgraph.constants import ERROR -from langgraph.errors import TaskNotFound +from langgraph.errors import CheckpointNotLatest, TaskNotFound from langgraph.pregel import Pregel from langgraph.pregel.algo import prepare_single_task from langgraph.pregel.executor import AsyncBackgroundExecutor, Submit @@ -22,6 +22,7 @@ from langgraph.scheduler.kafka.types import ( MessageToOrchestrator, Topics, ) +from langgraph.utils.config import patch_configurable class KafkaExecutor(AbstractAsyncContextManager): @@ -91,6 +92,8 @@ class KafkaExecutor(AbstractAsyncContextManager): async def each(self, msg: MessageToExecutor) -> None: try: await aretry(self.retry_policy, self.attempt, msg) + except CheckpointNotLatest: + pass except Exception as exc: await self.producer.send_and_wait( self.topics.error, @@ -103,9 +106,13 @@ class KafkaExecutor(AbstractAsyncContextManager): async def attempt(self, msg: MessageToExecutor) -> None: # process message - saved = await self.graph.checkpointer.aget_tuple(msg["config"]) + saved = await self.graph.checkpointer.aget_tuple( + patch_configurable(msg["config"], {"checkpoint_id": None}) + ) if saved is None: raise RuntimeError("Checkpoint not found") + if saved.checkpoint["id"] != msg["config"]["configurable"]["checkpoint_id"]: + raise CheckpointNotLatest() async with AsyncChannelsManager( self.graph.channels, saved.checkpoint, msg["config"], self.graph.store ) as (channels, managed), AsyncBackgroundExecutor() as submit: @@ -138,6 +145,8 @@ class KafkaExecutor(AbstractAsyncContextManager): await self.producer.send_and_wait( self.topics.orchestrator, value=MessageToOrchestrator(input=None, config=msg["config"]), + # use thread_id as partition key + key=msg["config"]["configurable"]["thread_id"].encode(), ) def _put_writes( diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index efa18d9e9..9abadf743 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -6,7 +6,13 @@ import aiokafka from langchain_core.runnables import ensure_config import langgraph.scheduler.kafka.serde as serde -from langgraph.constants import CONFIG_KEY_DEDUPE_TASKS, INTERRUPT, SCHEDULED +from langgraph.constants import ( + CONFIG_KEY_DEDUPE_TASKS, + CONFIG_KEY_ENSURE_LATEST, + INTERRUPT, + SCHEDULED, +) +from langgraph.errors import CheckpointNotLatest from langgraph.pregel import Pregel from langgraph.pregel.loop import AsyncPregelLoop from langgraph.pregel.types import RetryPolicy @@ -83,6 +89,8 @@ class KafkaOrchestrator(AbstractAsyncContextManager): async def each(self, msg: MessageToOrchestrator) -> None: try: await aretry(self.retry_policy, self.attempt, msg) + except CheckpointNotLatest: + pass except Exception as exc: await self.producer.send_and_wait( self.topics.error, @@ -127,6 +135,7 @@ class KafkaOrchestrator(AbstractAsyncContextManager): { **loop.checkpoint_config["configurable"], CONFIG_KEY_DEDUPE_TASKS: True, + CONFIG_KEY_ENSURE_LATEST: True, }, ), task=ExecutorTask(id=task.id, path=task.path), diff --git a/libs/scheduler-kafka/tests/test_fanout.py b/libs/scheduler-kafka/tests/test_fanout.py index e00787aee..4e5e9586a 100644 --- a/libs/scheduler-kafka/tests/test_fanout.py +++ b/libs/scheduler-kafka/tests/test_fanout.py @@ -126,6 +126,7 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - "config": { "callbacks": None, "configurable": { + "__pregel_ensure_latest": True, "__pregel_dedupe_tasks": True, "__pregel_resuming": False, "checkpoint_id": c.config["configurable"]["checkpoint_id"], @@ -146,6 +147,7 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - "config": { "callbacks": None, "configurable": { + "__pregel_ensure_latest": True, "__pregel_dedupe_tasks": True, "__pregel_resuming": False, "checkpoint_id": c.config["configurable"]["checkpoint_id"], @@ -206,6 +208,7 @@ async def test_fanout_graph_w_interrupt( "config": { "callbacks": None, "configurable": { + "__pregel_ensure_latest": True, "__pregel_dedupe_tasks": True, "__pregel_resuming": False, "checkpoint_id": c.config["configurable"]["checkpoint_id"], @@ -226,6 +229,7 @@ async def test_fanout_graph_w_interrupt( "config": { "callbacks": None, "configurable": { + "__pregel_ensure_latest": True, "__pregel_dedupe_tasks": True, "__pregel_resuming": False, "checkpoint_id": c.config["configurable"]["checkpoint_id"], From 7dc69a543d5c00fe3f59d5fbd552b1ec561c4d9c Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 14:26:00 -0700 Subject: [PATCH 19/34] Rename --- libs/scheduler-kafka/tests/{run.py => drain.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename libs/scheduler-kafka/tests/{run.py => drain.py} (100%) diff --git a/libs/scheduler-kafka/tests/run.py b/libs/scheduler-kafka/tests/drain.py similarity index 100% rename from libs/scheduler-kafka/tests/run.py rename to libs/scheduler-kafka/tests/drain.py From f7dffa023c7129239499cf44f0511e4ee9f3a136 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 15:16:35 -0700 Subject: [PATCH 20/34] Implement subgraph delegation for distributed arch --- libs/langgraph/langgraph/constants.py | 2 + libs/langgraph/langgraph/errors.py | 7 + libs/langgraph/langgraph/pregel/loop.py | 30 +- libs/langgraph/langgraph/pregel/runner.py | 6 +- .../langgraph/scheduler/kafka/executor.py | 58 +- .../langgraph/scheduler/kafka/orchestrator.py | 49 +- .../langgraph/scheduler/kafka/types.py | 4 +- libs/scheduler-kafka/tests/drain.py | 4 +- libs/scheduler-kafka/tests/test_fanout.py | 8 +- libs/scheduler-kafka/tests/test_subgraph.py | 554 +++++++++++++++++- 10 files changed, 691 insertions(+), 31 deletions(-) diff --git a/libs/langgraph/langgraph/constants.py b/libs/langgraph/langgraph/constants.py index 15dc47fd9..dd7efd6f7 100644 --- a/libs/langgraph/langgraph/constants.py +++ b/libs/langgraph/langgraph/constants.py @@ -11,6 +11,7 @@ CONFIG_KEY_RESUMING = "__pregel_resuming" CONFIG_KEY_TASK_ID = "__pregel_task_id" CONFIG_KEY_DEDUPE_TASKS = "__pregel_dedupe_tasks" CONFIG_KEY_ENSURE_LATEST = "__pregel_ensure_latest" +CONFIG_KEY_DELEGATE = "__pregel_delegate" # this one part of public API so more readable CONFIG_KEY_CHECKPOINT_MAP = "checkpoint_map" INTERRUPT = "__interrupt__" @@ -38,6 +39,7 @@ RESERVED = { CONFIG_KEY_TASK_ID, CONFIG_KEY_DEDUPE_TASKS, CONFIG_KEY_ENSURE_LATEST, + CONFIG_KEY_DELEGATE, INPUT, RUNTIME_PLACEHOLDER, } diff --git a/libs/langgraph/langgraph/errors.py b/libs/langgraph/langgraph/errors.py index 2fcd64473..ec84e0b28 100644 --- a/libs/langgraph/langgraph/errors.py +++ b/libs/langgraph/langgraph/errors.py @@ -43,6 +43,13 @@ class NodeInterrupt(GraphInterrupt): super().__init__([Interrupt(value)]) +class GraphDelegate(Exception): + """Raised when a graph is delegated.""" + + def __init__(self, *args: dict[str, Any]) -> None: + super().__init__(*args) + + class EmptyInputError(Exception): """Raised when graph receives an empty input.""" diff --git a/libs/langgraph/langgraph/pregel/loop.py b/libs/langgraph/langgraph/pregel/loop.py index c0551c14b..9d96fbc3e 100644 --- a/libs/langgraph/langgraph/pregel/loop.py +++ b/libs/langgraph/langgraph/pregel/loop.py @@ -40,6 +40,7 @@ from langgraph.checkpoint.base import ( from langgraph.constants import ( CONFIG_KEY_CHECKPOINT_MAP, CONFIG_KEY_DEDUPE_TASKS, + CONFIG_KEY_DELEGATE, CONFIG_KEY_ENSURE_LATEST, CONFIG_KEY_RESUMING, CONFIG_KEY_STREAM, @@ -50,7 +51,12 @@ from langgraph.constants import ( SCHEDULED, TAG_HIDDEN, ) -from langgraph.errors import CheckpointNotLatest, EmptyInputError, GraphInterrupt +from langgraph.errors import ( + CheckpointNotLatest, + EmptyInputError, + GraphDelegate, + GraphInterrupt, +) from langgraph.managed.base import ( ManagedValueMapping, ManagedValueSpec, @@ -337,6 +343,18 @@ class PregelLoop: self.status = "done" return False + # check if we should delegate (used by subgraphs in distributed mode) + if self.config["configurable"].get(CONFIG_KEY_DELEGATE): + assert self.input is INPUT_RESUMING + raise GraphDelegate( + { + "config": patch_configurable( + self.config, {CONFIG_KEY_DELEGATE: False} + ), + "input": None, + } + ) + # if there are pending writes from a previous loop, apply them if self.skip_done_tasks and self.checkpoint_pending_writes: for tid, k, v in self.checkpoint_pending_writes: @@ -412,6 +430,16 @@ class PregelLoop: ) # map inputs to channel updates elif input_writes := deque(map_input(input_keys, self.input)): + # check if we should delegate (used by subgraphs in distributed mode) + if self.config["configurable"].get(CONFIG_KEY_DELEGATE): + raise GraphDelegate( + { + "config": patch_configurable( + self.config, {CONFIG_KEY_DELEGATE: False} + ), + "input": self.input, + } + ) # discard any unfinished tasks from previous checkpoint discard_tasks = prepare_next_tasks( self.checkpoint, diff --git a/libs/langgraph/langgraph/pregel/runner.py b/libs/langgraph/langgraph/pregel/runner.py index 086afa9da..7a09afde3 100644 --- a/libs/langgraph/langgraph/pregel/runner.py +++ b/libs/langgraph/langgraph/pregel/runner.py @@ -13,7 +13,7 @@ from typing import ( ) from langgraph.constants import ERROR, INTERRUPT, NO_WRITES -from langgraph.errors import GraphInterrupt +from langgraph.errors import GraphDelegate, GraphInterrupt from langgraph.pregel.executor import Submit from langgraph.pregel.retry import arun_with_retry, run_with_retry from langgraph.pregel.types import PregelExecutableTask, RetryPolicy @@ -71,6 +71,8 @@ class PregelRunner: # save interrupt to checkpointer if interrupts := [(INTERRUPT, i) for i in exc.args[0]]: self.put_writes(task.id, interrupts) + elif isinstance(exc, GraphDelegate): + raise exc else: # save error to checkpointer self.put_writes(task.id, [(ERROR, exc)]) @@ -135,6 +137,8 @@ class PregelRunner: # save interrupt to checkpointer if interrupts := [(INTERRUPT, i) for i in exc.args[0]]: self.put_writes(task.id, interrupts) + elif isinstance(exc, GraphDelegate): + raise exc else: # save error to checkpointer self.put_writes(task.id, [(ERROR, exc)]) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index 714fbb647..f76c66052 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -4,11 +4,12 @@ from functools import partial from typing import Any, Optional, Self, Sequence import aiokafka +import orjson from langchain_core.runnables import RunnableConfig import langgraph.scheduler.kafka.serde as serde -from langgraph.constants import ERROR -from langgraph.errors import CheckpointNotLatest, TaskNotFound +from langgraph.constants import CONFIG_KEY_DELEGATE, ERROR, NS_END, NS_SEP +from langgraph.errors import CheckpointNotLatest, GraphDelegate, TaskNotFound from langgraph.pregel import Pregel from langgraph.pregel.algo import prepare_single_task from langgraph.pregel.executor import AsyncBackgroundExecutor, Submit @@ -59,10 +60,14 @@ class KafkaExecutor(AbstractAsyncContextManager): ) self.producer = await self.stack.enter_async_context( aiokafka.AIOKafkaProducer( + key_serializer=serde.dumps, value_serializer=serde.dumps, **self.kwargs, ) ) + self.subgraphs = { + k: v async for k, v in self.graph.aget_subgraphs(recurse=True) + } return self async def __aexit__(self, *args: Any) -> None: @@ -94,6 +99,23 @@ class KafkaExecutor(AbstractAsyncContextManager): await aretry(self.retry_policy, self.attempt, msg) except CheckpointNotLatest: pass + except GraphDelegate as exc: + for arg in exc.args: + await self.producer.send_and_wait( + self.topics.orchestrator, + value=MessageToOrchestrator( + config=arg["config"], + input=orjson.Fragment( + self.graph.checkpointer.serde.dumps(arg["input"]) + ), + finally_executor=[msg], + ), + # use thread_id, checkpoint_ns as partition key + key=( + arg["config"]["configurable"]["thread_id"], + arg["config"]["configurable"].get("checkpoint_ns"), + ), + ) except Exception as exc: await self.producer.send_and_wait( self.topics.error, @@ -105,6 +127,19 @@ class KafkaExecutor(AbstractAsyncContextManager): ) async def attempt(self, msg: MessageToExecutor) -> None: + # find graph + if checkpoint_ns := msg["config"]["configurable"].get("checkpoint_ns"): + # remove task_ids from checkpoint_ns + recast_checkpoint_ns = NS_SEP.join( + part.split(NS_END)[0] for part in checkpoint_ns.split(NS_SEP) + ) + # find the subgraph with the matching name + if recast_checkpoint_ns in self.subgraphs: + graph = self.subgraphs[recast_checkpoint_ns] + else: + raise ValueError(f"Subgraph {recast_checkpoint_ns} not found") + else: + graph = self.graph # process message saved = await self.graph.checkpointer.aget_tuple( patch_configurable(msg["config"], {"checkpoint_id": None}) @@ -114,17 +149,17 @@ class KafkaExecutor(AbstractAsyncContextManager): if saved.checkpoint["id"] != msg["config"]["configurable"]["checkpoint_id"]: raise CheckpointNotLatest() async with AsyncChannelsManager( - self.graph.channels, saved.checkpoint, msg["config"], self.graph.store + graph.channels, saved.checkpoint, msg["config"], self.graph.store ) as (channels, managed), AsyncBackgroundExecutor() as submit: if task := await asyncio.to_thread( prepare_single_task, msg["task"]["path"], msg["task"]["id"], checkpoint=saved.checkpoint, - processes=self.graph.nodes, + processes=graph.nodes, channels=channels, managed=managed, - config=msg["config"], + config=patch_configurable(msg["config"], {CONFIG_KEY_DELEGATE: True}), step=saved.metadata["step"] + 1, for_execution=True, checkpointer=self.graph.checkpointer, @@ -144,9 +179,16 @@ class KafkaExecutor(AbstractAsyncContextManager): # notify orchestrator await self.producer.send_and_wait( self.topics.orchestrator, - value=MessageToOrchestrator(input=None, config=msg["config"]), - # use thread_id as partition key - key=msg["config"]["configurable"]["thread_id"].encode(), + value=MessageToOrchestrator( + input=None, + config=msg["config"], + finally_executor=msg.get("finally_executor"), + ), + # use thread_id, checkpoint_ns as partition key + key=( + msg["config"]["configurable"]["thread_id"], + msg["config"]["configurable"].get("checkpoint_ns"), + ), ) def _put_writes( diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index 9abadf743..9d548280d 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -10,9 +10,11 @@ from langgraph.constants import ( CONFIG_KEY_DEDUPE_TASKS, CONFIG_KEY_ENSURE_LATEST, INTERRUPT, + NS_END, + NS_SEP, SCHEDULED, ) -from langgraph.errors import CheckpointNotLatest +from langgraph.errors import CheckpointNotLatest, GraphInterrupt from langgraph.pregel import Pregel from langgraph.pregel.loop import AsyncPregelLoop from langgraph.pregel.types import RetryPolicy @@ -60,6 +62,9 @@ class KafkaOrchestrator(AbstractAsyncContextManager): self.producer = await self.stack.enter_async_context( aiokafka.AIOKafkaProducer(value_serializer=serde.dumps, **self.kwargs) ) + self.subgraphs = { + k: v async for k, v in self.graph.aget_subgraphs(recurse=True) + } return self async def __aexit__(self, *args: Any) -> None: @@ -91,6 +96,8 @@ class KafkaOrchestrator(AbstractAsyncContextManager): await aretry(self.retry_policy, self.attempt, msg) except CheckpointNotLatest: pass + except GraphInterrupt: + pass except Exception as exc: await self.producer.send_and_wait( self.topics.error, @@ -102,6 +109,19 @@ class KafkaOrchestrator(AbstractAsyncContextManager): ) async def attempt(self, msg: MessageToOrchestrator) -> None: + # find graph + if checkpoint_ns := msg["config"]["configurable"].get("checkpoint_ns"): + # remove task_ids from checkpoint_ns + recast_checkpoint_ns = NS_SEP.join( + part.split(NS_END)[0] for part in checkpoint_ns.split(NS_SEP) + ) + # find the subgraph with the matching name + if recast_checkpoint_ns in self.subgraphs: + graph = self.subgraphs[recast_checkpoint_ns] + else: + raise ValueError(f"Subgraph {recast_checkpoint_ns} not found") + else: + graph = self.graph # process message async with AsyncPregelLoop( msg["input"], @@ -109,15 +129,15 @@ class KafkaOrchestrator(AbstractAsyncContextManager): stream=None, store=self.graph.store, checkpointer=self.graph.checkpointer, - nodes=self.graph.nodes, - specs=self.graph.channels, - output_keys=self.graph.output_channels, - stream_keys=self.graph.stream_channels, + nodes=graph.nodes, + specs=graph.channels, + output_keys=graph.output_channels, + stream_keys=graph.stream_channels, ) as loop: if loop.tick( - input_keys=self.graph.input_channels, - interrupt_after=self.graph.interrupt_after_nodes, - interrupt_before=self.graph.interrupt_before_nodes, + input_keys=graph.input_channels, + interrupt_after=graph.interrupt_after_nodes, + interrupt_before=graph.interrupt_before_nodes, ): # wait for checkpoint to be saved if hasattr(loop, "_put_checkpoint_fut"): @@ -139,6 +159,7 @@ class KafkaOrchestrator(AbstractAsyncContextManager): }, ), task=ExecutorTask(id=task.id, path=task.path), + finally_executor=msg.get("finally_executor"), ), ) for task in new_tasks @@ -162,5 +183,13 @@ class KafkaOrchestrator(AbstractAsyncContextManager): ) ], ) - else: - pass + elif loop.status == "done" and msg.get("finally_executor"): + # schedule any finally_executor tasks + futs = await asyncio.gather( + *( + self.producer.send(self.topics.executor, value=m) + for m in msg["finally_executor"] + ) + ) + # wait for messages to be sent + await asyncio.gather(*futs) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py index a63057319..3bc298b93 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/types.py @@ -1,4 +1,4 @@ -from typing import Any, NamedTuple, Optional, TypedDict, Union +from typing import Any, NamedTuple, Optional, Sequence, TypedDict, Union from langchain_core.runnables import RunnableConfig @@ -12,6 +12,7 @@ class Topics(NamedTuple): class MessageToOrchestrator(TypedDict): input: Optional[dict[str, Any]] config: RunnableConfig + finally_executor: Optional[Sequence["MessageToExecutor"]] class ExecutorTask(TypedDict): @@ -22,6 +23,7 @@ class ExecutorTask(TypedDict): class MessageToExecutor(TypedDict): config: RunnableConfig task: ExecutorTask + finally_executor: Optional[Sequence["MessageToExecutor"]] class ErrorMessage(TypedDict): diff --git a/libs/scheduler-kafka/tests/drain.py b/libs/scheduler-kafka/tests/drain.py index 6543dd0ee..b58906362 100644 --- a/libs/scheduler-kafka/tests/drain.py +++ b/libs/scheduler-kafka/tests/drain.py @@ -47,14 +47,14 @@ async def drain_topics( async for msgs in orch: orch_msgs.extend(msgs) if debug: - print("orch", len(msgs)) + print("\n---\norch", len(msgs), msgs) async def executor() -> None: async with KafkaExecutor(graph, topics) as exec: async for msgs in exec: exec_msgs.extend(msgs) if debug: - print("exec", len(msgs)) + print("\n---\nexec", len(msgs), msgs) async def error_consumer() -> None: async with AIOKafkaConsumer(topics.error) as consumer: diff --git a/libs/scheduler-kafka/tests/test_fanout.py b/libs/scheduler-kafka/tests/test_fanout.py index 4e5e9586a..0ecc3d80d 100644 --- a/libs/scheduler-kafka/tests/test_fanout.py +++ b/libs/scheduler-kafka/tests/test_fanout.py @@ -16,7 +16,7 @@ from langgraph.pregel import Pregel from langgraph.scheduler.kafka import serde from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics from tests.any import AnyDict -from tests.run import drain_topics +from tests.drain import drain_topics pytestmark = pytest.mark.anyio @@ -138,6 +138,7 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - "tags": [], }, "input": None, + "finally_executor": None, } for c in reversed(history) for _ in c.tasks @@ -162,6 +163,7 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - "id": t.id, "path": list(t.path), }, + "finally_executor": None, } for c in reversed(history) for t in c.tasks @@ -220,8 +222,11 @@ async def test_fanout_graph_w_interrupt( "tags": [], }, "input": None, + "finally_executor": None, } for c in reversed(history[1:]) # the last one wasn't executed + # orchestrator messages appear only after tasks for that checkpoint + # finish executing, ie. after executor sends message to resume checkpoint for _ in c.tasks ] assert exec_msgs == [ @@ -244,6 +249,7 @@ async def test_fanout_graph_w_interrupt( "id": t.id, "path": list(t.path), }, + "finally_executor": None, } for c in reversed(history[1:]) # the last one wasn't executed for t in c.tasks diff --git a/libs/scheduler-kafka/tests/test_subgraph.py b/libs/scheduler-kafka/tests/test_subgraph.py index f051d1369..a9c0062a8 100644 --- a/libs/scheduler-kafka/tests/test_subgraph.py +++ b/libs/scheduler-kafka/tests/test_subgraph.py @@ -15,8 +15,8 @@ from langgraph.graph.state import StateGraph from langgraph.pregel import Pregel from langgraph.scheduler.kafka import serde from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics -from tests.any import AnyStr -from tests.run import drain_topics +from tests.any import AnyDict, AnyStr +from tests.drain import drain_topics pytestmark = pytest.mark.anyio C = ParamSpec("C") @@ -140,14 +140,282 @@ async def test_subgraph_w_interrupt( # check interrupted state state = await graph.aget_state(config) - assert len(orch_msgs) == 4 - assert len(exec_msgs) == 3 + assert len(orch_msgs) == 6 + assert len(exec_msgs) == 5 assert state.next == ("weather_graph",) assert state.values == { "messages": [HumanMessage(id=AnyStr(), content="what's the weather in sf")], "route": "weather", } + # check outer history + history = [c async for c in graph.aget_state_history(config)] + assert len(history) == 3 + + # check child history + child_history = [ + c async for c in graph.aget_state_history(history[0].tasks[0].state) + ] + assert len(child_history) == 3 + + # check messages + assert ( + orch_msgs + == ( + # initial message to outer graph + [MessageToOrchestrator(input=input, config=config)] + # outer graph messages, until interrupted + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "input": None, + "finally_executor": None, + } + for c in reversed(history[1:]) # the last one wasn't executed + # orchestrator messages appear only after tasks for that checkpoint + # finish executing, ie. after executor sends message to resume checkpoint + for _ in c.tasks + ] + # initial message to child graph + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_checkpointer": None, + "__pregel_delegate": False, + "__pregel_read": None, + "__pregel_send": None, + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "__pregel_task_id": history[0].tasks[0].id, + "checkpoint_id": None, + "checkpoint_map": { + "": history[0].config["configurable"]["checkpoint_id"] + }, + "checkpoint_ns": history[0] + .tasks[0] + .state["configurable"]["checkpoint_ns"], + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "input": { + "messages": [ + { + "id": [ + "langchain", + "schema", + "messages", + "HumanMessage", + ], + "kwargs": { + "content": "what's the weather in sf", + "id": AnyStr(), + "type": "human", + }, + "lc": 1, + "type": "constructor", + } + ], + "route": "weather", + }, + "finally_executor": [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_ensure_latest": True, + "__pregel_resuming": False, + "checkpoint_id": history[0].config["configurable"][ + "checkpoint_id" + ], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "finally_executor": None, + "task": { + "id": history[0].tasks[0].id, + "path": list(history[0].tasks[0].path), + }, + } + ], + } + ] + # child graph messages, until interrupted + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_checkpointer": None, + "__pregel_delegate": False, + "__pregel_read": None, + "__pregel_send": None, + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "__pregel_task_id": history[0].tasks[0].id, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_map": { + "": history[0].config["configurable"]["checkpoint_id"] + }, + "checkpoint_ns": history[0] + .tasks[0] + .state["configurable"]["checkpoint_ns"], + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "input": None, + "finally_executor": [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_ensure_latest": True, + "__pregel_resuming": False, + "checkpoint_id": history[0].config["configurable"][ + "checkpoint_id" + ], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "finally_executor": None, + "task": { + "id": history[0].tasks[0].id, + "path": list(history[0].tasks[0].path), + }, + } + ], + } + for c in reversed(child_history[1:]) # the last one wasn't executed + # orchestrator messages appear only after tasks for that checkpoint + # finish executing, ie. after executor sends message to resume checkpoint + for _ in c.tasks + ] + ) + ) + assert ( + exec_msgs + == ( + # outer graph tasks + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "task": { + "id": t.id, + "path": list(t.path), + }, + "finally_executor": None, + } + for c in reversed(history) + for t in c.tasks + ] + # child graph tasks + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_checkpointer": None, + "__pregel_delegate": False, + "__pregel_read": None, + "__pregel_send": None, + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": False, + "__pregel_task_id": history[0].tasks[0].id, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_map": { + "": history[0].config["configurable"]["checkpoint_id"] + }, + "checkpoint_ns": history[0] + .tasks[0] + .state["configurable"]["checkpoint_ns"], + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "task": { + "id": t.id, + "path": list(t.path), + }, + "finally_executor": [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_ensure_latest": True, + "__pregel_resuming": False, + "checkpoint_id": history[0].config["configurable"][ + "checkpoint_id" + ], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "finally_executor": None, + "task": { + "id": history[0].tasks[0].id, + "path": list(history[0].tasks[0].path), + }, + } + ], + } + for c in reversed(child_history[1:]) # the last one wasn't executed + for t in c.tasks + ] + ) + ) + # resume the thread async with AIOKafkaProducer(value_serializer=serde.dumps) as producer: await producer.send_and_wait( @@ -156,13 +424,13 @@ async def test_subgraph_w_interrupt( ) orch_msgs, exec_msgs = await drain_topics( - topics, graph, config, until=lambda state: state.next == (), debug=True + topics, graph, config, until=lambda state: state.next == () ) # check final state state = await graph.aget_state(config) - assert len(orch_msgs) == 2 - assert len(exec_msgs) == 1 + assert len(orch_msgs) == 4 + assert len(exec_msgs) == 3 assert state.next == () assert state.values == { "messages": [ @@ -171,3 +439,275 @@ async def test_subgraph_w_interrupt( ], "route": "weather", } + + # check outer history + history = [c async for c in graph.aget_state_history(config)] + assert len(history) == 4 + + # check child history + # accessing second to last checkpoint, since that's the one w/ subgraph task + child_history = [ + c async for c in graph.aget_state_history(history[1].tasks[0].state) + ] + assert len(child_history) == 4 + + # check messages + assert ( + orch_msgs + == ( + # initial message to outer graph + [MessageToOrchestrator(input=None, config=config)] + # initial message to child graph + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_checkpointer": None, + "__pregel_delegate": False, + "__pregel_read": None, + "__pregel_send": None, + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": True, + "__pregel_task_id": history[1].tasks[0].id, + "checkpoint_id": None, + "checkpoint_map": { + "": history[1].config["configurable"]["checkpoint_id"] + }, + "checkpoint_ns": history[1] + .tasks[0] + .state["configurable"]["checkpoint_ns"], + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "input": None, + "finally_executor": [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_ensure_latest": True, + "__pregel_resuming": True, + "checkpoint_id": history[1].config["configurable"][ + "checkpoint_id" + ], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "finally_executor": None, + "task": { + "id": history[1].tasks[0].id, + "path": list(history[1].tasks[0].path), + }, + } + ], + } + ] + # child graph messages, from previous last checkpoint onwards + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_checkpointer": None, + "__pregel_delegate": False, + "__pregel_read": None, + "__pregel_send": None, + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": True, + "__pregel_task_id": history[1].tasks[0].id, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_map": { + "": history[1].config["configurable"]["checkpoint_id"] + }, + "checkpoint_ns": history[1] + .tasks[0] + .state["configurable"]["checkpoint_ns"], + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "input": None, + "finally_executor": [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_ensure_latest": True, + "__pregel_resuming": True, + "checkpoint_id": history[1].config["configurable"][ + "checkpoint_id" + ], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "finally_executor": None, + "task": { + "id": history[1].tasks[0].id, + "path": list(history[1].tasks[0].path), + }, + } + ], + } + for c in reversed(child_history[:2]) + for _ in c.tasks + ] + # outer graph messages, from previous last checkpoint onwards + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": True, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "input": None, + "finally_executor": None, + } + for c in reversed(history[:2]) + for _ in c.tasks + ] + ) + ) + assert ( + exec_msgs + == ( + # outer graph tasks + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": True, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "task": { + "id": t.id, + "path": list(t.path), + }, + "finally_executor": None, + } + for c in reversed(history[:2]) + for t in c.tasks + ] + # child graph tasks + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_checkpointer": None, + "__pregel_delegate": False, + "__pregel_read": None, + "__pregel_send": None, + "__pregel_ensure_latest": True, + "__pregel_dedupe_tasks": True, + "__pregel_resuming": True, + "__pregel_task_id": history[1].tasks[0].id, + "checkpoint_id": c.config["configurable"]["checkpoint_id"], + "checkpoint_map": { + "": history[1].config["configurable"]["checkpoint_id"] + }, + "checkpoint_ns": history[1] + .tasks[0] + .state["configurable"]["checkpoint_ns"], + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "task": { + "id": t.id, + "path": list(t.path), + }, + "finally_executor": [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_ensure_latest": True, + "__pregel_resuming": True, + "checkpoint_id": history[1].config["configurable"][ + "checkpoint_id" + ], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "finally_executor": None, + "task": { + "id": history[1].tasks[0].id, + "path": list(history[1].tasks[0].path), + }, + } + ], + } + for c in reversed(child_history[:2]) + for t in c.tasks + ] + # "finally" tasks + + [ + { + "config": { + "callbacks": None, + "configurable": { + "__pregel_dedupe_tasks": True, + "__pregel_ensure_latest": True, + "__pregel_resuming": True, + "checkpoint_id": history[1].config["configurable"][ + "checkpoint_id" + ], + "checkpoint_ns": "", + "thread_id": "1", + }, + "metadata": AnyDict(), + "recursion_limit": 25, + "tags": [], + }, + "finally_executor": None, + "task": { + "id": history[1].tasks[0].id, + "path": list(history[1].tasks[0].path), + }, + } + ] + ) + ) From 98b8595a94628a89fa211c53a1f988999391b781 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 15:20:38 -0700 Subject: [PATCH 21/34] Move to dev deps --- libs/scheduler-kafka/poetry.lock | 2 +- libs/scheduler-kafka/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/scheduler-kafka/poetry.lock b/libs/scheduler-kafka/poetry.lock index 14ac0aac5..0c84bbc33 100644 --- a/libs/scheduler-kafka/poetry.lock +++ b/libs/scheduler-kafka/poetry.lock @@ -1179,4 +1179,4 @@ watchmedo = ["PyYAML (>=3.10)"] [metadata] lock-version = "2.0" python-versions = "^3.9.0,<4.0" -content-hash = "c6f14b1de3fa0c90d7b8d860b85003b5b1261c0f64ccdf489777e213419bf278" +content-hash = "29e1bd946c9d7c9424219fa94fca03c80514d98c58e9fc5a7d52f735b8d0d8c9" diff --git a/libs/scheduler-kafka/pyproject.toml b/libs/scheduler-kafka/pyproject.toml index 8ca9618e0..49bc0b499 100644 --- a/libs/scheduler-kafka/pyproject.toml +++ b/libs/scheduler-kafka/pyproject.toml @@ -13,7 +13,6 @@ python = "^3.9.0,<4.0" orjson = "^3.10.7" crc32c = "^2.7.post1" aiokafka = "^0.11.0" -kafka-python-ng = "^2.2.2" [tool.poetry.group.dev.dependencies] ruff = "^0.6.2" @@ -25,6 +24,7 @@ mypy = "^1.10.0" langgraph = {path = "../langgraph", develop = true} langgraph-checkpoint-postgres = {path = "../checkpoint-postgres", develop = true} langgraph-checkpoint = {path = "../checkpoint", develop = true} +kafka-python-ng = "^2.2.2" [tool.pytest.ini_options] # --strict-markers will raise errors on unknown marks. From 270559b8803da07bc85518fe70e3076b5b2d9222 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 15:23:48 -0700 Subject: [PATCH 22/34] Implement changes in sqlite checkpointer --- .../langgraph/checkpoint/postgres/__init__.py | 8 +++++++- .../langgraph/checkpoint/sqlite/__init__.py | 7 ++++++- libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/aio.py | 7 ++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/libs/checkpoint-postgres/langgraph/checkpoint/postgres/__init__.py b/libs/checkpoint-postgres/langgraph/checkpoint/postgres/__init__.py index 1c064415a..6d5fbac8c 100644 --- a/libs/checkpoint-postgres/langgraph/checkpoint/postgres/__init__.py +++ b/libs/checkpoint-postgres/langgraph/checkpoint/postgres/__init__.py @@ -10,6 +10,7 @@ from psycopg.types.json import Jsonb from psycopg_pool import ConnectionPool from langgraph.checkpoint.base import ( + WRITES_IDX_MAP, ChannelVersions, Checkpoint, CheckpointMetadata, @@ -337,9 +338,14 @@ class PostgresSaver(BasePostgresSaver): writes (List[Tuple[str, Any]]): List of writes to store. task_id (str): Identifier for the task creating the writes. """ + query = ( + self.UPSERT_CHECKPOINT_WRITES_SQL + if all(w[0] in WRITES_IDX_MAP for w in writes) + else self.INSERT_CHECKPOINT_WRITES_SQL + ) with self._cursor(pipeline=True) as cur: cur.executemany( - self.UPSERT_CHECKPOINT_WRITES_SQL, + query, self._dump_writes( config["configurable"]["thread_id"], config["configurable"]["checkpoint_ns"], diff --git a/libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/__init__.py b/libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/__init__.py index 31bffd19c..8b6f728e9 100644 --- a/libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/__init__.py +++ b/libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/__init__.py @@ -435,9 +435,14 @@ class SqliteSaver(BaseCheckpointSaver): writes (Sequence[Tuple[str, Any]]): List of writes to store, each as (channel, value) pair. task_id (str): Identifier for the task creating the writes. """ + query = ( + "INSERT OR REPLACE INTO writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + if all(w[0] in WRITES_IDX_MAP for w in writes) + else "INSERT OR IGNORE INTO writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ) with self.cursor() as cur: cur.executemany( - "INSERT OR IGNORE INTO writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + query, [ ( str(config["configurable"]["thread_id"]), diff --git a/libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/aio.py b/libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/aio.py index 28ad76109..3ffd715c1 100644 --- a/libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/aio.py +++ b/libs/checkpoint-sqlite/langgraph/checkpoint/sqlite/aio.py @@ -476,10 +476,15 @@ class AsyncSqliteSaver(BaseCheckpointSaver): writes (Sequence[Tuple[str, Any]]): List of writes to store, each as (channel, value) pair. task_id (str): Identifier for the task creating the writes. """ + query = ( + "INSERT OR REPLACE INTO writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + if all(w[0] in WRITES_IDX_MAP for w in writes) + else "INSERT OR IGNORE INTO writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ) await self.setup() async with self.lock, self.conn.cursor() as cur: await cur.executemany( - "INSERT OR IGNORE INTO writes (thread_id, checkpoint_ns, checkpoint_id, task_id, idx, channel, type, value) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + query, [ ( str(config["configurable"]["thread_id"]), From fed6499a2e8b9e35278964433e0b0af29e1a4ed8 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 15:23:55 -0700 Subject: [PATCH 23/34] Run ci for kafka lib --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b64cfeea2..2251d9f79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,8 @@ "libs/cli", "libs/checkpoint", "libs/checkpoint-sqlite", - "libs/checkpoint-postgres" + "libs/checkpoint-postgres", + "libs/scheduler-kafka" ] uses: ./.github/workflows/_lint.yml with: @@ -56,7 +57,8 @@ "libs/cli", "libs/checkpoint", "libs/checkpoint-sqlite", - "libs/checkpoint-postgres" + "libs/checkpoint-postgres", + "libs/scheduler-kafka" ] uses: ./.github/workflows/_test.yml with: From 943cc4d83ae28911d212dde1adb94a5ea706dbed Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 16:02:16 -0700 Subject: [PATCH 24/34] Add readme, lint --- libs/scheduler-kafka/README.md | 133 +++++++++++++++++- .../scheduler-kafka/langgraph-distributed.png | Bin 0 -> 76422 bytes .../langgraph/scheduler/kafka/executor.py | 4 + .../langgraph/scheduler/kafka/orchestrator.py | 11 +- libs/scheduler-kafka/tests/drain.py | 3 +- libs/scheduler-kafka/tests/test_subgraph.py | 4 +- 6 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 libs/scheduler-kafka/langgraph-distributed.png diff --git a/libs/scheduler-kafka/README.md b/libs/scheduler-kafka/README.md index cedfcc53f..c279f5e38 100644 --- a/libs/scheduler-kafka/README.md +++ b/libs/scheduler-kafka/README.md @@ -1,3 +1,134 @@ # LangGraph Scheduler for Kafka -... +This library implements a distributed scheduler for LangGraph using Kafka as the message broker. + +## Architecture + +![](./langgraph-distributed.png) + +- Combination of Kafka (at least once) with a LangGraph Checkpointer provides exactly once semantics both for orchestrator and executor messages +- Checkpointer ensures writes for a given task are saved only once, even if the task is re-executed +- Checkpointer is used to record whether each task in each step has been successfully published to Kafka, to ensure tasks aren't lost, or published more than once +- Orchestrator and Executor manage commit of offsets manually to ensure tasks are marked as done only after finished processing +- Orchestrator and Executor pick up from the earliest message not yet consumed when restarted, to ensure no message is lost, and avoid processing messages more than once +- Orchestrator messages are keyed by thread ID and checkpoint NS, to ensure that no two consumers can process updates for same step of same thread concurrently +- Executor messages are not keyed, as they can be processed concurrently +- Orchestrator and Executor execute messages in configurable batches (up to N messages within space of X seconds), and dedupe messages intra-batch where appropriate (this is purely a performance optimization, with no impact on correctness whether applied or not) + +## Basic Usage + +Launch orchestrator and executor processes: + +`orchestrator.py` + +```python +import asyncio +import logging +import os + +from langgraph.scheduler.kafka.orchestrator import KafkaOrchestrator +from langgraph.scheduler.kafka.types import Topics + +from your_lib import graph # graph expected to be a compiled LangGraph graph + +logger = logging.getLogger(__name__) + +topics = Topics( + orchestrator: os.environ['KAFKA_TOPIC_ORCHESTRATOR'], + executor: os.environ['KAFKA_TOPIC_EXECUTOR'], + error: os.environ['KAFKA_TOPIC_ERROR'], +) + +async def main(): + async with KafkaOrchestrator(graph, topics) as orch: + async for msgs in orch: + logger.info('Procesed %d messages', len(msgs)) + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + asyncio.run(main()) +``` + +`executor.py` + +```python +import asyncio +import logging +import os + +from langgraph.scheduler.kafka.executor import KafkaExecutor +from langgraph.scheduler.kafka.types import Topics + +from your_lib import graph # graph expected to be a compiled LangGraph graph + +logger = logging.getLogger(__name__) + +topics = Topics( + orchestrator: os.environ['KAFKA_TOPIC_ORCHESTRATOR'], + executor: os.environ['KAFKA_TOPIC_EXECUTOR'], + error: os.environ['KAFKA_TOPIC_ERROR'], +) + +async def main(): + async with KafkaExecutor(graph, topics) as orch: + async for msgs in orch: + logger.info('Procesed %d messages', len(msgs)) + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + asyncio.run(main()) +``` + +```bash +export KAFKA_TOPIC_ORCHESTRATOR='orchestrator' +export KAFKA_TOPIC_EXECUTOR='executor' +export KAFKA_TOPIC_ERROR='error' +python orchestrator.py & +python executor.py & +``` + +## Configuration + +You can pass any of the following values as `kwargs` to either `KafkaOrchestrator` or `KafkaExecutor` to configure the consumer: + +- group_id (str): a name for the consumer group. Defaults to 'orchestrator' or 'executor', respectively. +- batch_max_n (int): Maximum number of messages to include in a single batch. Default: 10. +- batch_max_ms (int): Maximum time in milliseconds to wait for messages to include in a batch. Default: 1000. +- retry_policy (langgraph.pregel.types.RetryPolicy): Controls which graph-level errors will be retried when processing messages. A good use for this is to retry database errors thrown by the checkpointer. Defaults to None. + +### Connection settings + +By default the orchestrator and executor will attempt to connect to a Kafka broker running on `localhost:9092`. You can change connection settings by passing any of the following values as `kwargs` to either `KafkaOrchestrator` or `KafkaExecutor`: + +- bootstrap_servers: 'host[:port]' string (or list of 'host[:port]' + strings) that the consumer should contact to bootstrap initial + cluster metadata. This does not have to be the full node list. + It just needs to have at least one broker that will respond to + Metadata API Request. Default port is 9092. If no servers are + specified, will default to localhost:9092. +- client_id (str): a name for this client. This string is passed in + each request to servers and can be used to identify specific + server-side log entries that correspond to this client. Also + submitted to GroupCoordinator for logging with respect to + consumer group administration. Default: 'aiokafka-{ver}' +- request_timeout_ms (int): Client request timeout in milliseconds. + Default: 40000. +- metadata_max_age_ms (int): The period of time in milliseconds after + which we force a refresh of metadata even if we haven't seen + any partition leadership changes to proactively discover any + new brokers or partitions. Default: 300000 +- retry_backoff_ms (int): Milliseconds to backoff when retrying on + errors. Default: 100. +- api_version (str): specify which kafka API version to use. + AIOKafka supports Kafka API versions >=0.9 only. + If set to 'auto', will attempt to infer the broker version by + probing various APIs. Default: auto +- security_protocol (str): Protocol used to communicate with brokers. + Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL. + Default: PLAINTEXT. +- ssl_context (ssl.SSLContext): pre-configured SSLContext for wrapping + socket connections. For more information see :ref:`ssl_auth`. + Default: None. +- connections_max_idle_ms (int): Close idle connections after the number + of milliseconds specified by this config. Specifying `None` will + disable idle checks. Default: 540000 (9 minutes). diff --git a/libs/scheduler-kafka/langgraph-distributed.png b/libs/scheduler-kafka/langgraph-distributed.png new file mode 100644 index 0000000000000000000000000000000000000000..4315a01b57bbf5f3259d83a920edd702bacdc50f GIT binary patch literal 76422 zcmY(rbzD?k_dYzqFm!h+sB}m(AR!{4v=Y)tcXtgXASE|~bO;Dacf-&~Bi#c?w?pS| z@P3}>{l0(9XU?2CbI#stuf5i_uC>-gs4C0h}9tN=`OeNPf<>4H7UFTDxk~r z;i!xUm}F0lhmurOy5Qj%`PZVdvcq&kQheFrFJxt%hp63W#_!Jd(sfeIz0KWc%*L*s z4LdaaXjqvOtUOUVIqs%TCi?_ZH)7;)TrI<{u26bzvuau!75UoIoGxD>A-6N}=*USa zT|7U=B#p+)OIfhT=^uKNf9QH1F{K$bQd3fXp>R)~R{+xtVQT=_Bf;`~p2vb=j(9^M3(cU2A=ing!jHuTW4wk9ifxf57VwL(e7Up8Q zwHD^Ki-~c~X1a-$c0EIHMXZw}B92flF3$ODzklDPeL)Tsw6g7g$(He1Cwp52zZcMd z#{5buP`@yVq(D#O6_8D}o~&+bi&5Qqh0o`4C(G$8`EW6R3wPy>s`20W_&XEf4sTn{ zb(*Wj&u0bSC{=%EztT(Eqkd=Ju}YE)aV#J)?Oap-2zyg<&- z#^|VPUX}PO{AWczf!5@JqWWtaI^XlvvlQ8QH@{=wyeWL5pOBt9SF8~a6Pv9qGGu91 zpbyqP?|UnWJ4OvNj?}W{MJ1Lj2?cC5JQ9iGFVhvS*k4!32_v9S(MoFgto6SI{{+%R z-(3@%J`YY7{sV#RX|*NE{Oqc)=X+fpy)!kiHDBVi=gze)%cW(&KakScK44hmkNPO6 zOQO_7olN`X@c3{iMn}z{C&&Nw|3;jb15YdYHL73yD$qF(VHX>qWn$sH2(GWM7k0Cs z^BfDLPo4as&!49kZR`MRbb9?ES=7U8y{Mie-idaNzutWAX4o)*Ju{It{(tM7EAzv_+Fn+N9T`*d+8@VyG`%R^}JC@H=AIenW+Gx6gXc6&&0k~&`ct{x+t zS4Z5lLotod0kyuDY?x=)+bkcO9{WJEORP)I;YlD~S#2xowJiNPA&$MiNH2Qlpl_Y^ zm0i_8;K)F&G`#yUqz?`fCb;_M6kQZ$MXxI?vi6IgAfaQe@rkR{RffB&PT28fzHM=; zh~Ee)Da}O%tBwTuCft53uHrY-0E_k__U@O@!^}ze7HOvSU}jUD7#Gy@jh>Yx ze4UV$P*FYjfNf#f+(^3W2fG-ce`SUcF?}_DD0S?_m1vg}$BYBZQ0jwt`?`J@7lY8SS00CBJFiYp4`4 z*{3lLjM29*aNfR!U`mSae;)sw8}IGrpj-Miru6F~tgdl@PYU0Jd z9`Vv(%q(uBnRR#~CY1wrO`x9C9M)#ph}!$bdgNPPU|I8}5uVY)^;N(#)+j&PK1GYC zkZZzYP-}3E5vK@f`j$X8iKd)K7y#4qH-V*Rbi+^RFuQ@U+I~RK&!;2YfG8Dk(HFaN zPRUXNC@B#2YX(N^=o@#bz>?xtW|%I;%u6-WcD!RP*v#q!%iB}G^SKn#;r0Zq~m##!9(1ES4_J!CG#>?j|K6IWsO1d#%FuA!EY z3`l5sSN6d8G6urQ7wa+z3ZQR%9aybGYzbDzc|6@RgAz9&{YV$F`Vi@b7|yyy+DYYL zYP5|SQJj-%?*s$pvT|O*Ky%7sAtqZJ!o6}VG2WxfK!1JVdW&PG`{}TL1s%z9=D$b# zj<-^fm6a9mT6@h{T!wcK$t@>xm3<8|I_6SM$yI z>l2*M)yUcAa2#1eYmb~P*gK{s1UgFrN^MTa=Z63)a8MM%%SWrqgU?5VK3l~o6u7PrMLswXY5Iq&g+p(<~ z#u0*H$-2E?aqk#D&IJbpP?e0Pk1Yu;Z2-fqV0++rf{@T15y~dlUkG=iou2Pst$INo z&NDizQIEFIYs!Ik)bG|xhRw1&eid`-g}&dBp&aj=1GEAUK3My4xgIlk?#yC1(wt)q zKzzYR?j7P%Vhq~QKBe$Kd?Xv00FRJAei5sN*^ZKQ^{L&uhfU^l4RjIiVY})fUSf-L zNDir7$SMonF>QEOk_lauFvZ*>l+5;2IQf7R7icjhh$7A;pW?^Xm@# zQpS+?2g~O~R3sng_;w|W#gecKoQl>Tm~h_U{TMFfIEnv@13ng^9V*l7{2=wL_v~e! z7au-~mXRa!(WPw* zk)q$rzmGqM;vlLcKQtzDEujFw#OIH`<9pj#?3Rj%ShPJ3nLhZfEh;tVc0Myu`2vz_@LB*}QY2kP%TIZ0##>Ag#0(Fa12B z<>KkSY+5a&v(_%dCro>&d(<1^ZXLSnK`{pUUe8SL(NrK42a(qA zf7n^k)-Uvp(cu_rT0`}Z%7>cV}VO(-?n3A!jA{hqzQjHpDNl@`5-# zkHHkTO8y3q{r8k0YvwOBD*FT?CJ&GG)MP;)7r5daZ!ZoNX;*eBUbrdb(fX)iDyA1u zy4|Y@lKqg@;DWaFr_XmNeUC6g!w6w8nYZ#+$m$wWV)}XPtu$XiW-5W5ot`e2<>>?Y zO9M}-lV{eaWzNTl-ibh-D5X1SmM(2?%P!ncvkqruYsI1Zbh7SE?^4dszaVGL09Z`Y zck_50kSle>p^6_HEqNAFVs7CcLw&V|SNWbRpyXjCj%sYV;eg{ww(|JvzZU&pydL-f zQZ+`N8}%zs>ao@X35lUJ&lgaBi(7WoR;`O-BgM(!!}OHo?eEvgvTId#V(uy5T@sU9 z6N?0&HGE6GnLG5j%04S}_lTjGZ8SX{erO%Q%d0*eVeO`j8d zT5a=3+A5(#$o?m9<(-bP`E`9eU6UdI*g=$xQo&ON@RFK)x&Eh;aI83l z9Hnma_FmxkQ25nbG_rooUdsLjME)9h>mE@%5w}EHB-A>nYu2rD(O+eyIh8A1Z1RhC z>gn1=Ac?4`m-%QPzB%!&^zpUXj#5X1pki3!$>3Ix>kn;8^_;kS9sGOe)9)bXhSTr! z)`~c9=^&>z3nQP3&(p_1BCM_miK|nOu_iUHu`215M$*aq$RkPBJ##8PKGj0Acy49i zl*_PkhEu=#XO*qqoZAD!onODk3p!g33q|-fnE(3>)l0N6*l%pI7~}(;ivjfUC3~rL zBN6W~dqN2YekVr`{y5Q9yqTTQ-(JpyPAR;0V~fteu^aB?qpjUVNb)WkPcCOtl`hob z+X_dQrA|=QzH;9g8BcTCy)~Ov;G80rP5<(r*{DX4qajZOPs;~x$VXk^zIA;q^@^s9 zPR44IAT4qpe&Xo5&ZWNrlW-mS^@AmqW&)5_^LD;Qn@jX(?jxt`9dDYNttBabic2=DS-p$ggU5I2*A>t453buK zm+)&en*aEY+fyo?zHJ_G{W7xWyiWgQ_RD%X-=VM#8h z7G^#QcVs!`8{A&ctG9hCXA*?ny9;OYk{GuNYk z$k(nrSdU++;UwnsIXCo;O{g3GHL7TlywF5uiro1A=KOFc5*zdGHsEROKQk_ubRj)K zjp9~o4?QtqqKzS9v*42{O#N4N=~>&a-{<#lQ`onBe&Hi+!V!J!C)n~UP52lnqD&Gg zX@vUHc~W-68qv_Nm|h;PXAUcj`7f`+DhuJ3nu~9Hb;xarZJ8f5)z)uBeh#DqSgHid zwRx6dHsPoP?MUhfMjj%S6=``1Y=JIPj8c(r?-oSvu+9N^-~g~5=h$@E8DP##*8)5n zb^%36($NMsAh{6woToP!$l8bUA_Qp=%s_UeDB?&xmIdG-mUGdQkePC4-OnZU*~C=T zW<1e*0Q9=e58ho~u4JRdEOm3^4kPIf#z&GO%18GhO`2LkTM{MGY3^j_#EzT?aLfm3 z>9L)X2#P~dd8mF{aFzrW_h-M&godj8C6mxYlaC*HSelRzId~I|hH8PJ^u*QT;_pue57e8txjI$Hw z*lkjHX?1$0d?==kRRWj{T90UDj9YZ5;XZgu5I5I0Pn4Fb7-_m|QFq4GA3`fUHd9Ir zU%kZHg%-(Piqm{l+@;+ZWPYHt^eJyQDXpZ!g=?JjKib`h^Hz2$1M2#I2#61$bQJi& zgUOb?DnyrwA;A5l&e5Az4>E*73DlvK$`6jAp#04kybZ@C=N_m(KL9XIEP*y=Vjmq! zSmNpcXZzs+8+Vv3U}iwi5mnYR3?a~I^jZl}4=_q>iQjZe%oG5ESsMbld#^w+TCE_U z5muI0kD##RcjSpqNC6PFh>01z3M}TRgNg&D8Da+!z9Wpz6sG zH{c?$rtqC_+xMhdAL@(tK}iGLA>g)`;54?U{y9SdcyoG8r1lQ;8{bNK)}Mj)`la#s zDOw93*Q^X+lV+2tl*pZPTeO)2iiFeMT)La#c;BVFgU62Qo$8(6;IK9q{_S(Hu> z`vY9f=n=9hLO|vErjFtwc+o+mI;^}|47l+yb}E=w`qAond$U7uH+|T(a3HG9#Rzi~ zT;xO$b|ns@*`g!7@?am0&#Y6R@C!tih%O55#@mPK9zd0^0Wp% zp0s8{bU`Uqq7izI56Th)0hHimBj7o`qqN-W%Kl^F&j&BkV-8zooVYEw>W%?dKyIP+ z;Jd&UBxf1TiywDdX&X#&>-r+j%K|`KC_X|T?zbU)Qgs0AnWHVc`3WEDu7~4RTfham z!>2$o(Kk8qCy%Irmg3IDU0wWxF&vULI|+7WH{Y}HwXq9Ur{LZCIQ#ZZgwrpf2w5sh zDcK8&z;!IzMQ|8az`<;#Qpg*ws7Ha?l zEQ76wHws0M7ukYSaHEZYoJ-2c z4b>XLm)aA~Hn1rzTJH#LF5$_Qfiv-#}WPwvMY4BC62c#H-q-o9@c&LkFNfI0} z2V45uhlgHRL-B7AS`U7SFV9!sAgg1J{X4c6c4<02xjgqQ)7!j^sF;a&HPfzmJQ=6Y z_O`IXXgxS`Gx5$Ds!kZfc%w4|q%-Aw?{YsNtCxKo1X+bC4DG#t#N6}1Psn=*rj20& z%6xmYk10kr&SeHRwf;y}vIK}N(oII}MElujor%pg3H|cVI9NQ*rl(C}Y|X6m`TAb{ z!5u}i6H$=fI%x>*c5vHEge(oEq=9Y-7Nwa$#>Vr1F05P;MNWY}Rnf?E8PY2Mf6|s$>H=99 z#`EcWQDnUUz0N7HfiBg@G9Fi~A3<#J6lY__=}8`hjUJxMvM^Q=!~}4)B|Le?eU3j* z4Eq^s;PMG~nwzp%;J}k=L-4&}(TokbLSaVlKo?W?AS6v@lIoV3EJCQqzc$p-bWYA^ zWjv(&sV($p)C*qjc^^14up_=r4M~OcX!ukW*7eJP{S{gv%nx-WHWAPoKB8#n;G?03qFl7z3-4%%U$v4(_A$ZnjWL85p3g@o<%?= zJQlFD3JyGL4zZjO64f`mx-9Te&l=!Bio`q|zTM$M-3;WJw~t?Z8y58&>^Pe@y}kCU z@!{r)%}l?VoX&sXLU+v4ev#$;s`L*$k1l?C&VnJ*`|XN}SU{X_hPsqpL|6ddKq)>V~!6I>x6*ekOjXJ=rKFSA`G?eGWzT`sRvx z5@S$P7U&(<9-1DWCj4~Q%iy#hf$0?!7vdIm7^5Hwt1FEv;k9<=!{tKlOCavaSm^F* zJi=j^Ih(Bx@H<3VJ(wC1(@N0Z+(w5KG{{-cz=%nXIdoM?k?i$%OgnA(_8@Kj=kI3s z&}0*RwVszM&s=M8ObnU$OZn5EZwp}`1L?**i3Yz`Q`+4^67u6~u=H?ZAj>6!|LAZ0 zBiE{?(sEW*T@^SoI#;6QV<>c%GE+r>b?8TBmw_>%_1Vj0?sSOUXE0}w)IsuV^OCLD zY46iT=&nzo@<`~ZX^2VPncYZTepZHAR(*bpJ35fh54{q9C3i_Qf~5x?!uS&GY=VYM zgL&$5c~Uc)S?+rc-~2nezgbl#e!D}^DxJ^>4tkKDd-#2McCz_wJ%PA!Qd;GivqxLZ z^jAngvj0Z9WsGD~u-ERb?m#s7DrL!B$pbgwdNFnVc(ozLMHx2GbUu)?PP6-dw4$5t{24o2Z+hdC#M~&r zKV|3?_^#D{aG#p2P!Y#5aKOPzp~eB}N>ZnqghK=OM>N*0KF=Vnq^V;l!oUUY=^;Wv zloT6qY=nc}aVelS=m&fWqZY#utdo0*a@oDr!5w8sx*&Ah>$J0v4qEg*f7PFp7eTc` zei8N@(IP_4A6;RwK;aZ{0tn@9W9xDpQFj47<>YGwb{4Hs=2vRUrJgXXTB3ZDj;?us z_@zIv&hc0$;6(|>xCnOH!MG9ACaA8Xy=&DD$z|mBe55+|$Qy4nozaaRY@I;;F-+`G zCTeJbD@(`|10%Xta7Z@Aq+J2YyQ{WsPzq_sQ&SZ&DY3pUsOpRNl=(b47clVi! z$GC&r2^=^iiQX$`^Wr0q9&0?6+4t=o$Pk_IJzdtWG()|tZPA6E@v~$)Wn9dK-V7Fe z$>e{t%sOfiU>8Dt*VVIL0?iPpT{x?o+b3L#;wZQ)9F`PdNq4WQU#__t3@Mf1@5uD8 zL(+k5G>dt^=sQ+ewHjKWI#D?T=2aH-4GVniOpreTEH5m=3HmbOk(tU!OevE~EJWz; zcck+td`oI0;&Xai%rihrd;qH_{l}3#BYNcXbLleT5vB|(=@-|D<>=ni3uSXHxTGxe?K#RVZ?;xqH?tqapWs-VyEf|{u>V>yij9Hbx z1+9M)CY82PI6n2n(nf8YjA=4CVvEohf%SnKG89Iwwo62=rAiufUd0`Q<=Pu@#lqw# z;y9DXL7vDV`=`IGW?v>KI5Pt|pr7!D)tr0UR?-MJ=44yg6?4hARClaw~=NH#8fI(Cp=D|s=LTq)3TvEzk8)w2S_i^lD3r-x)n z#*3H-Y;V5_#pr!LhU(2fh7-UeDQq$yItmwH>&$jy$*r$G=NOF7mu#0l@u>*K7m{PY zQYZv%H)dZ``p4TvuNpPZ$3;$@<6i^ek@@Y~Zl?BoY$F)P*0h70w)h|aiJ3NkOdFi|nG9^gsS&jBsLF?MaXbm`gj8P!xq^+9CJV9@T z_86-^tWW`hKBC9zsERTdKD~rppLAZ9_vtLRUqh3HCsdOHRA#|LR{q?cT$UN0sA)&z zC5P+l6+xthzLWMy-G$dUwPTS#?xpyOPdF+$#;u-TS@AEYZd6JRcQI|C_YKiisifF? z@*PJ~g;`gbV2a_K=I|=wn`SW#b10cs#bJzK`Pyy8gS3^;F>&6th0gPE+0$<_IzKlz z>B^qg)LiL~x|aW2^1R+{`l4CbsmQknzrLda*B68hIl}qNJePedox9Y>$`$+#_YM$~ zrMfmWLfl_6w;H2W;@)e)BeNvMtN)~Gm##7W$ro-TT_3`VFdyK5_TWwKCHQ0A-Eeot z>VBUNZCjie`=t8=qVbZ!Rmng2J#&FLHrGO_+PcPIpj^uq6vW+rVdwEkmu}1SvrD04 z#*50lUa9v__to_XxiJmlE^RN66AYuENTHth>2gJ%BJECHgqXXGr2S&vR%@}`UE^x) zJ3lf2cEb~v`8q#!d(}Vs;5%C3d9^=CyCZ&-Xq;#)K=&I<1OU}%TCwscw`jaBu4Ka|dL@!Iigsxph?&@S~i zyRe)2I)hxPD4daZ)YPBO+;R{0K)mWVz(LIdQ-85aEIOQ~A)2pdi*+Ir(2<#0#wf@g zW@z=-0cPtHM`K!c@Wpq+^pZYXS9u12tdlE8rM;Q=?)>x5e|P<91gI`ySTR}r=Yswv z1>sB*%I0?OG#z>V=*gp$fY$S>E3p#e*ae$iQo=@hXOTYNRf!e9lfJdaXW_ww&7021 z?2?X?o7~cc%RlF)?VpZ-bn`z`u6jz>#|Rf^SB1^PPDCcJMJ*DOB~Ikd?!0bn&g1vN z;v_4iw&bu^f!btA5sK2vl$5bcUFr%U;&I!tszvQ-mk{P8{aH8_kA+Yx27q)3|zU z75@2hztpW?d#aVd+PhKiR`2loyO_51!BL*%OMIHtcooSWynsIsUAB`q>g^^SMn=*c zJ>}vPs>fU<4vAbYXDA!iYy-U0Jsau|e-+M$Svkr@*9u4+UE4f)sKyvwqUwFbj))kF zXzNl{i*5z36syeNuv(euUfYp4$i?sJLQdD*VgxGKMSjwg-fwU-sleXCJiZZWH)(Ww z#xEbhh~eb7l=Hf(Z9a{z()$F*!=q2=g+-r)6)(5MqUqkeRfC7M<&$B9m^o~%R1J>A zWagM{E6@GCyX*ezgZMidajXump^EvVb{TQS@9A0lQCx zkOhM-VYO5*-jbs~bd~@-;?A!)O7n6xUG5C0dhX~=Vz#}$?z}ykY}GkmGQ8HGL;^Qb zaIf$M+_oqAOIt(+{EC{mGIfd@9!}B-SicY6k834njw|vHemTt+Sz{3MHh4(G8RFl5 z&J(T{yyDy!E9>(m$WfUxK&EqvfV^w@`O~wX0jC?#xY>GQFG=8N8uz4z-s($@gvElq zW{J^Gto%lg7VqKAz`k=Mu)h%jlo2edPbB>W@=+#Gd#S2xG;#Yc#~Y#nZ&dFv^y{IE zxZ63I%5<5Z0~^wvWTXl=yNRJ@=kE9wA{6%R$8$Pv&A8965{!4$zTQ`gkf=&}d`bjM zNEbr0s-d-L2X>c<5^u+ClS4k0+UT71!WvO{huNe&suqqWD?4jEq`*?Ip*d|j2*^@QI7m1jU69)*6V>~i$sW2>U58Q>TuKJt9otqdx{Bsw!N{_rzO$!q; zf>{oRqdEG(X>?-lS)ub^Lj097P{MSzB2YL*F2nn$o_CkqhepFmg6+kQdFNqt@D~=A zyvTCYuboWBN)DmtEu9T9E7$HQ#uigSNZ zu>9eVA(hHW=Q|ZuF%u_yppg$6vKk%WZl>vBV*+Zn1Y-m()3_Y>xj$hnLgOwM$Ttpv zgn*i16aEqWC`{Ly63k{q*8}xN5q%@qL^K2H>WNPaIE{E|5n|_0xcPtqTjb?zYSxoF zMo0fL9{*M7EGy#H0BajK6pewfX@;cXyTj!GduBw0w|_a$tIcY(RF|yadyd>qfQH~% z#E=#S?k;c0FN8@p+0e6?F9g1@tfvFze`fp9*$CDUP()~NVG(+($p@-yJP_mP_iOU~ z?*9IOwls#lX3tOXM>YnTWf&|qY&);xMN?u}FT{wW7g7pnb&3-sb9|^A12$L<@*_jO zPbDCt=i_#t3M_tkip`S>5U^E4NIltz<6vTgz4l;Gc8|69vW@u z0kD9xA!?F?9bw_)`_5^#FD+JH@?r{r=$NZ3PO(9;Axu3hy+X9`U@413+H;B!7x}u? zmz#C5m>WTt@Tm+JyuL z#1Bm>UR6;Y1BXAv93b!OCbW-xQR{X8LJ5@E>+fMlAm(vM4b;z(Df-Qe~c1wN4 z0Fl8q47km{SglQ#&R;l=qMsYQ}DaVR(xGV{hUiQQcL8Ec^#H61gAkTId z7)O%BC)^*bzIgz4Bpz&Ne+MF2ZBGw>Sd8;W08`TIk8|2?>l9OT{DuFaY0Tt7Xm-=M z4Q*EZKLiEw=z<$h;0v3n^^NLhRJItd0?I%G-QN+1mz9GVaN{}dt7@9_XqOF5-oKQX z+iR%pmmz$+$8{9D7~hc=Wz=}!WzH03(r2NYSy24|m70Yhg7+qE$ypLe0>;(*j2X^% zbpGl&&bM8(V(os1uO1%owVX1dA};*b)%my6c@)EL8j`YW3585vM$cEmdCbQ9hiPsz zi?s)8$X_q={S!7c*#XC`%tp#S;7CQYP4-9LabfMQPg5#8P_Ah@UVWDyPdpbL=G9Vw zbz{%B;fqEDS#P=*5^oT?Z~`*=D(&ctEyrq1w>CzE4Z4JmOFE5~{TkOJZ#|eI<10OJ zO(U0(=r56Emb>6bk{(t5V>5qfd%72-ZCr`CL8G(P7sD@&9*B^SEPK&-4hHNMOo<~d z)Hy!K$-!9s3!_6EfW!2J1=JpxcNh~=G+9TpOZcAAav{hx5t@)nOW0A0?hYJ&mon`V z7vr${`zi9kNFoXIuWwUFCkaEz6lc&`F)TFClkUC0I!pNM+7fl{QkoJ8#p7en@EYer z=zQ)hO1D3 z4fYL7k;rj^;ZcPwKMYJjOzO#zdlgUM)DtEW0c8h?56mZXnN!^}P?%0Lpg$I4+>$9g zZ(*{x7EckvxkRX)Pu5oU5^DMw38m@r1BXNXb5Q({5w8qD3qY`q#|TnK{uX)`v69!}9Ku^4E<_9=liH{{{wO(g<8b~AGRS$|Mb1WES}g^3jzfTGC8A^)3) z579#Qnt0RRE){X<3MEe&(i(RM9F^Da1hNd;ojtQR2snYDI)qmbbK6lzl2tnOguR=K zt4Rpon$QTt0CW5LE48!HKZNsM>B)jO7H64^lAcre>z!jmLc@1g^iy6m7rT=4Om70N z3GAYI30Hyi!t)a2egZdDY?5_14ZA_{N>?|qRYs;*zEoicfvID$IiX5nIqvZ0Aqq-y zJt;egbS>8M=OQ|GkHJISY7<3G4~e9; zk~$jS@nv-gGM1Iv0O!7zOX%b}{qZfjsW~af8$A7@hyUa0R*M+R|d}n=1wY9KU>?+JBCyx$*n$>q&Y@Qv#X_z5?!XBmOkKWC4GxyeUVr zAiQ^glWh_e_PPNhCw39yRjy_9QNV$I8=1;A>Y9c*ff`|mBDCq7!0BVibr|tIP!834 zP<|-;_W6BL22m>LT2i+votpaF+(Wq}Frmb}>%Mo{y9&?1M&{G2Qhr535%X&6iKpn` zt?sUsgVnHBhQMQ|;qrr+Vx|j^`!ZLAYkNwyys(Glah2$*1LbliTPa_$POWfJYK`sB z=kdwq9eKl|l&J*i4N2Z7G)ZSYF+%gnKIcE<_9fNV@6KR*)7<9hFzK7|>NoqD_7=`e z$#~gYP_&;}z2`S7cePb9{UCJ9;V?6x76FMhyepwl9Vi_Ya)>LlRc!XOgGO)0GwEP!fkN` z*p$n6AXkCSM_$p9zB;+?3|(F@-Bf8IODHUe3my|D*l5!3IU)j`9mqDx^}llaJzX@q zl`Ye8Mwbq?k=~m{iE~4WrJ+(AaVB47s!4YN1}1)!1rG(+PmtC9;AL9?$vZ=uB>deC zAbqk2pq~eGl3OZEbQ)6#wC`KaS(?x+5$9TQU_2?^)4#o(UTUz=+`#bRIxW*x%8J^E zDGB$lnUhAFIawklSv0~3R|3Ms(a_w3z6gJ3Dh16&{N7j8D2+3BcBy|dJEpF#&a}QN zsZ>)F@WtMI_0nx$Qu5YqQlKNfG>TqK(e9db?d8LsttgkvhL{<|>tVj{o_IHG78w@% z{)cRpXxic|n~px!kv@1rWz)lIXR4U1Vr-&HNlUXUbR!*Uj_A^2M_O=E^0;S*7$ple zQgM!@bLOPyxCqdVP%A+7ng}qE4k#!Q~Bqt^A8*732h76_wIVJ2>l$yEixX!K;NZbyFIxR?xAl|vjN4dE zhkuQwTI}`93kHP4QW%i?+c}pPJ}*WudktvV*z+t%6JF~~kH1Sm8pOw@T0(}Tl&eRI zA1c&u*TXmz*yjpq;&LRgzw3ATbI=~4BF~#Dv=)#}$HvIvjZZi~UC%$}`{!2I{MIlai8=P! zB4^j$$7mX17YF@$xcg#9aj;jt9t%=yld;vtzW!8Q*!PKhbBd6I6(2u8p2xs#8h`(u zjjwOh#nd6o;C=cda4S22g zB2M?0DPY0UJ^H<`v*}CwA4h6=nB=+m&l{Q1JBUG`X& zSHqiPZ~QO*S_TJ?Sn@^A)4!k=q8=0Y=d8W`u0Ekqs4;iXOl^whiz;N zTmDAIok4E6J`#-cw9Bq3Pu(C>u)O;EzRRz&a`?TK?27?8ZN5^U*-{SOR&d(V(K~s| z>Lf_TR>nh)gy#J%Z$h$5!{>yhlF#XYPwZtpK$*W0PdSBde zp5>S)Ik)|7cY*(;x#Va2-+m}&D;S_qI8Y<{F!f+^=}b*-G4+NX3FdwoXu0NPgJMI{ zA_X0uEgoxkEMCA!LeDQ$8Ct|Yy7~68K+qJ|wNveXrb)!eSnH(wQWIS?eL?tz)&8A& zp@wiLO(vCPvTwfiyyf0*}Z z>SsH9Z4|adv3+*!cLJBj z1C}`pt}lA8q0=A<=ob9+fhUbGsNl-Ph)5f7?jyS2K9T{71>K&8-}sZ~f^=vRXHRy_ zlO^EpWLYnZ;@v8jdH_0){5+LP!2`q2XF#1t&37MAt{*L>*Sumb%M5OZBX1EYTmiHs zi!~7<&dN$qy4B`v5rV)IRp#55{WLxPeyk*04RhD0y*7s}(a7G@nb^q;0SFofAF@8k z?`QsiF1D`VK>l8DzU)uSb8h&^OYEOq$r*o1o?YEF_%m{k*1`ZU5tP*JL%5_q8J z)N9W@s0YaPkHkO#p(ntX3?|eudipqR(ax!bJPVy;Uge{?#xlUzEfv)Dry&^vN6WAb zb;p^c5)y#v-sE~uhkZk&2s5}Y*clwe-H71khLKR0U{i#wz|f`&`3BBt`)_4pA>vTi zJHWq#KfZ-Z(#+t#sc15uvz!gkJh|7~%OH32o+De&>}{a?rWQ(dsV<9UvQ8wAS;hcb ziLjP}!U+QPi#8sl5!3;iiAR!=++%uD7QGHr4+4i^>D)RN2-y?fm*Ov3b`+$b;oZ!X zYc=}cE}|nnX3yXRp;JuN{Vj@s>u6Q5BPEjRGSJaPw&F^1iyPteFGT=JIEannZ$qd8 zVe#W%QpO}8gdmuztGkH1K`Do8histr+_`V1N#BfLky+Sr7vaN zB@yUsN!;=Bg>G9w+ZBB3!=CIf+I){n$F+AojBsVB(37z+~62O{|XeFc^2IS&MRO;5iGDxr_o2gH{@Lj<1~)diK9aO}ehCay@O?SP9eO>OWeGW?(wQ-HH#77ZAhIj<~Kw~mncw=f6FQNT3J0dh? z`zMIWh%DL~HwUS-n>?qcnx(j7;(Y&o3Co-4stl?06{y%d-fP zRNjCgrP40-*uhLwun4BUhe~2lRxr_((~|UWgco7M6fB6 z9__S}Ecg0P8_FccnVWUmxHKHYE@3<==v~1)IsYcfq#t9l{%a1g>0)_B>tq8{8FAUZ z4t#1NMXNEZA^5o54V?y|#ntkcV7FcWUe(O9i{BC<@8F1?gt-nXmHiPm^iMhmXUE0! zNY@2)t--~m&_6N20H|%eU)*0EKvNR{G4y`d zV)i56B975@5Yw{b)e)X~!J5LI0gg>L=h1t30N18XDaZWP*HXHmX~}O(7)5IzKyJ1{ z8zRKZj(Zgc^Y5B+zm~w zHCs4pJD*AIQMs_`siD^k)-YH-|K0&zwXj>P5=1F)%cQfHG9_65&UvcCz5WHf_^O9K zQ42_k$P98a!p;Iq;P6UPUhK4G8i{XW*aMWY=;p&D4;_xKiRQ^0DbM%J9#sJbYH1EB zdOU!85JxWLkh0Au;B+@8FFN?++^H5L#-8H7f>B&!XtyvVLI0ZQafa^G8;Z9Xc50Ha z=eme4$(;_* zr6d>N|L(zmB$e}Tf#Zv}$eNzk!g7AmgTVRu`BYe~UDFmKM#*NP)LmPex)+M*5TUh#*3-Zd zwXZG)w7VE$@Ttw~D)7Kv*jWfqTDYg2RF7%Ifu2%lJyt&dy72zj@?rnTaeEij^%%x< z(tY1Npsq(<`HJhny@V?0VSvyZzE%Fu7YyQr1*rE(!R;|jPJnCg`*Qiq7wn8^mEl`N zv+vm*YndjC;^iU1+soqCplg?Kx0en3;s>=paU;0L$C;{nXnVPU29r&HQ5I1=D46lC zLcv#kVrOM#@c(u$pUZmw+pK|Z0I7Fco8^j7Hdw^WBNL%n#fj8kit`@t`S*XAAWL(f|D~fNVo& z^_2-xFE40~wYr7eRpQA`uJCsg64Hw`$BU}8H1SHyE=+!pqgi}hEn9h|SMo_MCKjjt z%=Y`DKYU{fdYco+G)%YNmVGAMjC8MeLFuzq8_uLCwZj=9P2|qLX0%P+9O&>2Z6LlD zeB(p=2{Kl63%J4{>B^tWM^B!lrKZRG*rHoSgxW7&+015{`{sBHD!qR`$*^K8@8IATozZylaM|^_tdwf?sNu-tgM?{U2og3I!HKvmOt&7jl?0dg0#$?dn8 zY^M~T$Jh}Lj#vR=EQLuphnIhzCFq7PJ55Eq1ZC=?V?Xozei3t?XxI}-Un@u^Oj%w} zw_j9*o2Nwgx$nHRfVD&dCo2tBLr+hZ&#$(d&Aor68ao z|2{~ez4)W2W2p2yQo^FOHPI)+Jj1`==d;t+Vzk%uzk4V8gw)j3Jb5qq-3;37*e^BN zS)5MVC1Zzc$!w)ZU}NJ}xH$U{4Z3_2JsLW`OkMG(hE?TLIBHJo%ZchYR)tf*@Tt#D zr6q_3-rI*Q4gK4u_4un)Z}tUtLOiI^>Ho3!RZ(?qOPjE93l0H-B)EHU4-%Z<1ef6M z?h@Qxg9UeYcL^l8ySr;Ia!&3&|GlFh`mKA6eqjJ>ueoN`syVB^uWHtw@l#82)UR7= zb{8GmG(}79>En<>Lr44ESz)rb!D0Vq{im0n;)r#se(|@z+DMeUJxNXiga!IbGYM=% zn-wtw;?%D;^XrleQ}s9XNECJ4)Le#)OeI#S3kGC##@5jTpHGIRc-S!Q_<`bE2h8&2 z3v5y3+twzcfMAOc1iY}`($jnb&JXWEgfNxxnP6pl^`@%v+XhG%&g?g@F@EiGVj0jw zZFcMy!e(Gcs{MT^O8{P>3yzl~QdD(-svRF;CbIrWHb_6!bKB_}4beW@MH9kan{Z z=6XCg*iO~)8{fMw&YO~FSFgtxOjG@|;>dsnBrK?!z~>~qnlxB=D<5gw86UxKbI94sRsTXx++#llZj zlN1M^k-+hotCa<9b6aukKy3L^gQd~!^$7Pitb9J<$z%oR0>fvaa{-hO3cP|{0VQo(f`H78 z6FMT4C>WiEQ+V&)>?pq#mI@3V%qhT8LfA0XBkOy=^0F@juYQK08S8nD^O8o`-xB;9 zvJ#aErOo?PWq^C%ikk|N*n2j@F9eiKV_}z?cDMiu-gV?X!N-W!}@h_CJYc5>SVUemka<>=MWq7YY?Dpoi%@G}9+K7)i2p=QtY{DwaXkAth< zw6~Via-tsSz0t-U@_-Kq*F2v_|5%sle2zl3k-BoZ@-9JVRB*9TO&;@fp#pINQ?fmNZbPEqw*F7T*mC6=1JT+gS5qeh^!5|&V)63Uuy6~-_ zyVIgH=kn5x4P0H^W^L7c)7AfS>xyax46gNi$$RYf=FJea9^ z^C|azuP|m<$oGgv*{7j>_E5wg@--(3ieWbg>m~>tx;dj$uj7`bOukGZokIf1gG6)O zh^St|0zT|4Gh%+JXK41;_)NZjaP^#tSJ2#!U`}vW+~(ku;b2o9LS7+C;RamnP$b~- zme9>u1>E?rrxqC%+6f5*F0=%cv(um{^VWkV^!_aIY*sb*}S8mTyUcL!;=-Ggw;=BeAavNZ`IOv{Mdud&VF zmCw!Y>+%IAGed-)Z$PzkpqD<2mAZ+X8vMqOm8D?Vkq<%=>rvJ~m4O9RsgVD=p!!x3 zH~#4|w{gb1VPUASbb`8BsT;q2+0yxZZeQ65$^Oh z9;L6+5GO`P*^bpm1(SBOY!JxcIDLD!GFAL(H8h`aCN13oFOInknx&`@=LW zq$flWl}@)4Ck=wm!6e|jubw5pp*I4*rMohb2T^9rt9r54*Kc5oD+o+AP*-%nhK#AT z-+Sx?-RQ5d(Y3c=ZHGbn?N;BIlcP=Wd=2B)v1BlMuoX<|ybKKQ2ts*7kKL?UwExv= zb%4|3G^avp4!nJD34aw^fb%tIW_iCoX~kxxMr<=_?DWf3k27tu>*zVBzRzwJi>Ip0 z)Z!k}1g+7MouKf!0D_GF62}SI1jjKQ_+-1f?2L7|FwW@5v-c;TtM_~NH`qqA4cd!T zxjf0V+z8fE_iF@qxu=l*eeDpEEwglwI_@=Bnk%L-y~Bv6u2oqyx^}ha;T?;^x-G%` zoVYW6Z%Bfs!k?Kb7Z$kiE)gN~K~Ea)K(2dPxlClq4e=SS(3?d>`=PiA@1zCmx80NF z7oBO?hN-~EvO!sh1|D{fJ-yXQ-34OARCp_aF3Z4=wPkA&D12xaAf$G&b|@{9l%N=m zc2qChtM?~ZE4l%f8qmq!>+M$M(BoUa)+;;=AQJGo2CtCUbdqD>=BKA;4Pq5&EBE;_7c+ZO@60r8+d+WCC3Xne&FJ-*MwqaEy^>C ztx0d%_O6^j9h|+itAi~Qf*rZ}m64^FFcFj!?I&l;R;$&BC?|l+6KcXJKa>P06Z9xH zF~EUzU4UmoWy(*$xWO6m`2cvi-Fgz9OPW(n{UXLnTDf{ZKv!qvstI%S3M{oP$_!V! z6w(na{2Uqu*$7=A(t?|?C@my3jKBDpd6ZcVUsgq9(Z@*#|L6Ge%!xz*r8Zyt^*web z8HEWKg_?E=sv2}2$RqC(0*)xz|GxPK%Ku7Xoan+?>@#&8@TvmdZqhJX2 zj;zqi7F=zz(kHY8Ww)Pu)#5v0ci6$tK7TlQt-UzR3~pt%N@Mhp9m(;X&vHE#e{3IC zKHpNnvmB+IHr$qZ2g-;ZyjD27s{%@vddn4jJQ<9RDM@<76Ljefc=GePv-7@WzEP<2 zdoQ)h$C?!i{_`t(K7;9^HiC4x-}cUrV-f?@Fg>^RQ45Ki$ zDu3rX{LsEHe2<86<=$(PD$#so=i+M@mLsGbyYBk+^5#&H#EM|jI#0 zPGY0Ys+in&%gZ*v`V4Vpp#eM;)aAvx$4P|&ch`KwHF}PFHh0qa2FXa^ag49%cMl<; zf(jjeAwu&6c{Wnk@^zlIbw9({P_Pzwk?rZU`je@d*V~_z8w~a1htQwgHtWSKp?11I zFw;RFLqEhf86*x%8sZu2jP^t;ASXrZ4g_)E`^E^~qFXgXhe8Nou{>oyNN)R-J5UH= zzG)dn#nQT@vICcD`Kr??u-T=*Nc`2E53IXzn*h@f%?H>6*iuE! z5)htMN*LBt{l#H~3)m;@5vT_DXK)k{22sdb=q5P9N1@uFcDKf`=~~&2nXd-abnI#j z3W@nUahA_h>zc1Lsl~aL&Wa<#%hHP6#wtdQur8cCu9fS@@-v@AO+~l=La^F|6zzPx z>ch?>ry?j@x0!sOT8o3R1i{-z{V58INprk35aVSp{R?aHZO%X6=C|W()7HcHQ{{mI z^;F8hHMfJGoIl4e8DiSLH~zbsgDf3JBy+hXU5wol8z**NTx-9Zf)0GF)e^DXGhchv z$}pOR#IJ->5KQ*%y@1QEgI-3p=G2nmz5ck#hrjZ8IbK@e91tGD;$(Db5+cgKsGxoy zypL1xwz}?A`b5cM_SdoFf1brZr3QKq+^{N!{YeJ?zU6PKc*71;EbD)q-T(bZ=FJN{ zen>79`S0<(fg~w>j_lu=#eWS3K(>dMZVw((irFvu-~VwP=*GpN0riRE&uRUCU!?E= z8mVEnzP%Vq*}1&H!rfB*Ytj5c5n!CLaafOaqcF?ouY&I8^U|1}tpoCZ{< z(Nk7n@{fiLf8UG&q-u)z;s0x}NKj81Fo(5i1=c_93jQq_{0qTB%D@~> zO$%}VGNgYC8wrAXQBkDe-@On1J=jYdl>f_L6Iw`{_>02%4P@KB<0jZ8iZsw+sB;;M zRR3ionQy2w(@JgJPJh}{$|_-~G}z>-)|$;)EDj6}F>`QmH04tM?q&6t+}?xV0p$)l z{bKz>DJu#8@N|EiCy_vxr&y$vC!a69b<%|lAkp*i&-b4#TMVQ*cXL`Sb}NVIn8)~@ zA2g>YTisombvpgpUEZ6`Xs0`GCAFV(0{!X!7=-t?mu4H^2Mba$zHh(-ryG*dWw?LX z9WO1KEj7;;Yqc$@J8#Bqoej~m*=Yw4IbrKrFARv0pRD*IzdD%O`&BgZc)d*Mq)Gm- z5djl_X&;EG$N?}Saq8ZV%kDr&5QC)EuE3N+#iHb#9Fpl(mlbcA;ZA=94SS^$H3Mnh zTYW$leA}%g!({XM+J@a_kNXmijYuKMt^lMFVQdYBm7E`p;xaOhlVz=UdpxNnTu1d? zsg6s|0bFK%_>QT&(`6aPKfgyc>=hQlTUe&@xYjnUD^-1wvOVp=PEoD;(qPVev*P>Z zXrWhf;C ztjXBrPp_7LudV}TsH5g{X_z|XGiO|#!!Wj`6$tuNWliUvBu2e}$w~tWm^y++XB>~C zhV9fOW<$pUhMl%CNO)G&zjkoUY?vQSz`B15W6vC zQmKH6E$mK}q>LQVs8)8XHrf^J+p$_K%-FB&ko?#wCM6@|oSmJucmTBRpjK_P$|*Tx zZ)@e6of7_;2*39<61i{RZZ%%)np>-UfAh+8G?n9(L*IfOr{)ZlE%2YvHyjhHt!@n| znoFfWG%u8;r!|A}zx~-WAK-Vc1l!&P1I4%iC5_K%$6%F-_$GGo%<*vUS119`oRn|2 zywxk|XxAp9HZ zAvf;}$3HqAbmAg*s6ccmUa+g}$fvl=xUd*wmKPN4B&cHqJ_z(3;rw;6_iO0n{a~&6 zJT7mR+&>!}eS6s~DGGmA`5akhC9|4-IcmAqysp!*v#=Vog6b;;7}6xV+2?SCv;AQ$*(yJ1yGAqAMicX%u1;L#D%1a;llTU{&@6BCFQ)GUgH z3f4pSj)|}PEOUqXo=%aktY4qQt&{SkQW}xhR~ z1X`58!Dr2@XnA_PPdS{g!#i?md49ULNV#6Hjlf~h_DYFBKFSo@VzX>}O|~Gl?#mj^ zxktflJVDY0I=lYAJRCrtTX}ByVLt_!u6Ft#`BEPoHykvGiSn|q z18THCf$UzaT>WuA3QW#H;#~A5rS$G{pUY}4_4a)GUWN;>u@ej-a$;h=2-_$$6~ZC4 z`@_2BH-L_^<8~dT z0ZQE5vBYQhwK@9)N3tL%{oh-4CJMY!ei4c;Wzm1@p&Wi^*WX`J^7gkUjS~VtbWjZ6 zFP3}h&d`7{er!)YeK}?H`=anC{&4;%efhV4*1m|Rkc2hb&Q<3p3@xdDca{B1YT475 z0Oz!5_&W)A!-!AKSSis-&5S$Ur2LcXC0 zrVoE6YQc_<6vSn{2*1#b3%3K`6%MvTfii=okjfF0{{XrB?iGg6YG7z{jD0HLF`mcZeeLYXcD~@iGQ19sNe_nAG^NK6w_8U`V{*S@_H;>t% z2VH$@5~|u|TFkejaUp6KPaq(&#r$gme`doU0Z9h6sCGZQ8f{;A@=cjMRjXgkRz>L3upU!vd;fB8_<*fiJw#)stg8t{BCaAJC4i#|{wEC$*zxCV(S3VKf z)|cTr_Loc?#Qalqcds^E)H&As8h*c#+hAbkwoAESeiJzou_t(;0l8Pi3&=v@|rYfY0CLW!28d&a*n5tT3(F&W_K`HMZ^; zaxfAst<4EQ+R7i*F-Q#U5bw==*YbyH^)^iB` zq+_m_(;46xVc0Ad#D*B`skq_ngJTf%Kn*h5EA8H-J$yl>YcO`o)`+l(h;E;XRTvE| zEsuRlOG~Zn?EG$YU```QuJN+BaSsj-PEOt)f`1tZ_PSZ~y6FhIxpC_5?k0No?i~jS z3CR`U`IOdR#JmZZdxkyYN;GQK zJLyLz8`0Ho$J;zTl}{V?sCsj&UN?!XaWORN4@AG><>gg6Iy$nlv5C%LJk@|Z)hLis zOr3^=3p(ZxIu^W5iURcl;Tokr;{(L?dN3Cb)?PZun zM`42_$&U=_=bzsoo7WqX&uCw$RFX{L6CzuCAzQVo&>^01rS~?LK9$pcuN8J`&GoF`{2^yZXT zltMeBM6HIZ1LN*Xs5^?%y2;u(>()RF`63`XJ}Ix3#hOZFEHxHcCIToYRHNC6QPu4z zyXQyO;Gjv`&B7R_(s9gO?3j4M`wMn@^l=Dpa0uw17+MS&@c!90Ay~AjCcg^fED6J^ zCUWQD95?!-wwoLdgE6PNp1;z^Mkp1lSWDKiZnRBwZ`ah__-M)&d?Kwjo1I9aQKQKd z;HTl}JZ^g?xCed+a)T3+dU!o*t@Q_`pXq62F9 z=vBPD)^s)iM32N={iqv%l*onD*22J;J?~K=ZY5I9qK+S_|YIAvYSA~TUOK^ofp2?Xm6`K6u%qJrrJ7H zrbVwr(qoRAk>HUxFW5`^?9+Kv3TetGdK?liUeFyQ2wUrU@YbObb&m%>!P5rd@C2N8-$R^0Z9+8GREzmCgxKi&1dinBpd#h6R{D4X4Gfvq z&-AL58y;i>>}m)g`UEr&`YXW+$1LBfsw%8pYbCNw9`BhMmAg_&6j$D^ZulriR#lsR zz@Q!)AjvZ2L2D9uMKcJMN^Tw8WuNOxbEOg3&N>10Jc4uE?Z{5V(|(2zPF_OQBIDck z2ocR< z&52L36;6NVM%9<`X7r(rVIzVGw_(G@>G~%9Rb77xJu|WxZwzx>&tw>~NlJ*FAlZ6e z6y({hmtdS;WtGVbe3h(O#A_=zF{2t}NH8)zji!&HXMGfk_74inQUbS=%+dKh(c+54w=~{Hv+_AE2 zBjf}Xka`)gYQk^hz_0Nr6bobhtys+oQBY*}y1$~gF`ys91_g#|VNfX;#iC%Jx+Z7}twQY@SaQtVaiam+mjXpM7arC$}f)YP1A5pX+S!DuNl;{8?E4EnAf z)&zM3woNKY_|GG^h({!nGgQe&1KFTrmNr;?udOz_I>s`GSdfi)Bd-S$Eo@_xA^UZS zJo;bZaMD!WbulV2{>m7=%PE#33 zKrDpDniB+F3Fki}lM@Whv#zbaZt5fSOX#Gt`cS`T2d?kfgR}4S1?~ zMw-#`*va{O+f0YCop;x7TLMECJ5#RMP3O3hky+@CJdBdr+VZjE*G+sn9M|3=f382f z*Jqe`is=d2Dl>Yzrr$m{<t7xl1 ziaJnRR!rrox7OrW@HB7I)c9Tw+sp9d+&?~g-{zoXeUDD)G<u*9zmUaO?trX*cVd(N%7w$ShH5C0ImD3(>APBu z$+r-Mw(z0EKE}E^>2segn30%;!mgi#x!8o0uJtn#Sp{KcPY1{&ddXw-T03dbAZiU` zs}m^naYFD@l8woiooSF$tajo|V;V;1$3a%$FH=Wt=MtE|x$y)<)NwAMhqSq5_P+0s(PsGgN z<^FWtlvas4I{J7x*obm+9gAlT^#HqVi2a?x@bECV6Mcepct)$5)Vc2#+z7V&FXQ4% z*g%%aWH!t6{&e^-d&q`G9MsHP$@Ao`x;=NLMg`*&4)k$!a8uL$(=~_zSHX+!l%gN_ z-dMD1>^~lUX?*RK|72rh)0LT-iQPM?YhkR3n&~WfPFl9d7T_{4>*eq7f0HobQ=;C> zpuHdx20PxIcS~EW@hdL0*p8CqrXTX+1HT)XL@@n2XN~Ff`$_pYbtLq09B)9;Ns|w` z&c#)NVL3!uW>eUvm$y1!qe7K6-^Tz`PiQvt&6=i-*A6$H$oU$VRTneg2WcbjwN-b+ z$DlR<_$@)7h!>*->fpxX-mC2waFzzZ?Eos*O77#NpE*IG+1}kXzA4$j+T!p|IL}c} zXY1z2dvX5Lsj0oIdH6?*P2VgzOIcZ7vYNP@Q99`G742WgG`A<#0-#Cu}i}7{=_W zyiY?NqQbus)qNj%QwuC}zb%CI{zNY;hH+IVXS<;rx;`fGNuZ)Mb&KmV2nhdil#jlm zqBz}M7^k(jx08#ED_Fck?~ew1F-%o5rizp`(dwHc7PbmaYeySjuo3iR`Ubh$iUfe; zGsHV3s1oS}wYLd`xid68&Z1>HOP{B0@yOhmSzCk1GPyvV-Re>*a8AUnN0Bxkoj-U1 zlicPmOfND3Nw$l@1!0R#pI=>tV@bq@Rl_Wpk!A9NCvUSX8!rLT0xz5N0Aw{8kw=aa z8X@n~g0WOILcb}9CBm3@+n%f51Bk;z!})8XNhQuEB(m||TUbfMHK!i`y6MxBhO^-% zi)03QH>2_yuCOQQpgM6xKp+(Xi8h@X9XmrU7;=eV!gim-cqk zCz1aOvK!aaQB<_=0Nn^PZ5i3DI*_bOz4f)5X*1YUlt0{tZfRY z%aj5AEvw9)x^gQy5wBfmr&;_l2LsqTIsbX$R@LdXpDO@)ym;Y)t`j&;i?A;~rtS{{ zO8Bj!xM|{!%YPs=q6`!iWK%yOwk!2SnShigO4xC?eCIK7Y#Eq8pyfC0a7RTsR-(@= zYpf?Ai`<6P9K8mh&E~DF74Us3{}Tx^7ywt2&US2;6TMK;oo-Iv#?x!18zArS=IZ^y zSTFMf0A%h^q-fxLFXR~`I2RY!LW?W5oqD~M4lbHqi3{I|RO=r&`tosaRN&+7(0(ah zkcsWoAY#$*SZshB_b)3e zqcdU>0QaK4!evW>+h79)*M!`SD3w0Ezgy7yGfM9Z;$9~dIs!-U$A;_;!-J8Lkr|Kj zf5DHxp{+Kk_e1+aBJPkmutJDvz47kv^THTTi&~2Ii0?M6KBPGDDK>2{uWS4dd+`hXI)~BlXeq?E$8UBpkl$AFl}v|7}a5;a;_EPQ@Q0XhTb<7qkI! zY(8i&eal$cn|@77WzSzgg}=J92sYWK60os>K`Oo|c;9vn*XzL`v$!#vH($GMnR-jl zK5w^+zQLk(XLFPMk2j%;I{tG)ad>pJ)HfyNbNjkb$fsqe$)+}!zVUP zqBD3}%NTDo;8CE_`jo9}n=F$pUM=aS=c!dAeqwo&rnbu-WsSmqer=%6Ws7inP4HUO zH!L$Q-2XtXS{GHpOl1&OZAeNV*hGKA8hx|7qyb=l&e`UW>kBEK4P;PGuC97FhZ0K0 z(s}kPEEYN2V#sB#u!5(t0dT*Zf=h9PjNU&6na=&O^Wc_GdpIcFsm?G(0Yig}wJ$6s zD)H|2liS%U3XMuLK?DK$+CvWt@*H+VgYU{Dl1Fw5WJH5N|QCGIZBJvSMszfaNxw)R$gBq5Ld)f?r z=+p7V8^x4E@`%=e1jdKB|K0lyej__ zLaA0MC`7tFY-n}nX@vLZ@h7BFg5jz5fG}Xsl1%ne6HlnBI)`<1yoQ_1lRDp>tujnT zdXp<22ZG89a)#>}+CLxT%W!zSvriF?AfRP5wprjHi;RF(M?mU(oN=l)R(!6}tTR(X zK+pFGGo9{2b)ZHYYMu9COPNLumJ)K~boH+eD0#ZlE5K;sGkyicJTyZ$n{|&jMr&Y^ zYa+MGPvue+A9%(u7YBmEq}rpMV{!$j8qe?Roor%czpI}fMeg2M33^K6p3~)d)MDtn z#`$ZM>!+2!F?kvX$S@lDPhfJewgP< z45iS@_CDo4)yAIR*X;Y1g9hrG?VlWoNAqx`A@^;)Uc`xBbg{WzB9=h^^iPWkvq^~{`=C9#^n>gt z-jTQnWbgZ_MJ)f&!LcxQ``FSi0Et5UWjIkT2S}8v)@A|Ath}NEE|J^$tRBEu{z2VM zmBa09Nx+hW2RUXR4>>flw>|043=c~#xt^RSbd>R((cstv2{F)9baBZ(9%6dX|jW=@5-{~KMod(Yo<{5OwlvxUuksKV}2Q}jP;%Jo! z>i=w+J}XhH!e1A;iWYx^`HjLh$1X~eU=mxN94U7DArNi@+M9U(31%5>so2UGJfT>1 zI~+ROBBjt6gYMV6k6G^V7l?sS$6M|uMN4b8=roWUR$m%qfW_5Ic1k0NDynJCL*v_8=npOE5pEMX83zmuEr7$`mP9gL6Ox%g2; z*jZBCC3rsS2N9GQYNgq)Vogm=Xy|5_=;Iyj2q2&p&keDdvY11LN-G?)TAg4mFlxZG z=}pK8>s;H!TP2vm;JK3#n3&8L|6v?1jz@9!Bm0Ab`DnC?g@0gm;xvuzC1K(=dIN%x#<18V8P*FC7w~Fd0 ze?IT!ep+U_gPB%n_rSgF;LIEvuGwdpv@{=s3!q*jI^l?=1 zo94f_=D{ltCGC>4|P`zSa~uo=iURurzzxq>r3;vT+b_ZQi1J zK>b{M+*Li`%jBk|JkN%=(nMIduH=OA#Onm#)i)A9`1uQaKb6S+cQKA^gOEPyRH4Us z28wH!ca-IG~%x}E+KDe^Gg(t`1m@~k|z|emjqHp{Dyi8x`I=GqtcDz`XnZxM?8sE!<*C=xMBD_=yXa`LMx^%~k^o#+pq93&J9u3s%IrsgVl_-o}kU71c~a^|^J zs&qH?J5ZQYD0UKVFsz659(PW3f~rMBuRJaZSTJFp)xm$c`QABTz;D1-G#91OG^uSe zIW@VXt7?jvOC2&~Fo4s-sdLyvkQQP5s*HU|d-PD?jz-JqWNz0WMiCZwdVBMeHMVr% zffdyE71-#;6Zsv)6MCU+px57K?vr8yNoe7bWLAI#KH6#@PGV-X-&3GbuSWsEN&9!X z>@&&RcRhM%Gp!S|kg6%g#Wb6hf{Okd^H792yWEuQ9nbSV+4E6P1dD zeU63ZPXw=bntj|pt(aF-kLMfdgt_i-9r_^jW%%}XjP$9lKcsrU1Y((V-pUa<%%j%w z&4JWCq^iBw%wd>I2)L&ALTki z7rT?9y~wyMuyq!>PHCn~%^cDhytMPr%bw5P#b1B|ln#Ja4hITnUg#+3A8hhQ-2n9m zzzh23D;}(HvUIV{q2*;lLeB@rGm|lFN>H1?&%4HfA0jUP5(7f;gp}$e)!331K|g9tcuk0DtrePyCfu>eB2yqO zG=lvCGd^1=MbQtoqGKV+Jf~WdJh8l@dBJGj3nYyeQtVRJ-Flm0+V%!`&p zryphASTBl4&AyW%uvgBFxwI_i&BVsmce>uUy1MG!6M~HpVHr{G`Roy7@C}8&EuQbB z4XB4mq6cyfmoR*k5O#~X64)#j3$MEZkzptexJz-y_6>&O2S|OZ&14r*(9wze0yC)w zrw?YU>H)$;4<;B3{x$xj+mL8S`3Hw~RyAXCZ@V*ky?=uk zK(2|>F6Wj!^fl4jTYzQal1X9`Hb)LvBOQo{j*b>v6+#{t79g|XcYDr!g=J`9KpLN-`cYWWsWc~gviZa3 z+(I{ZDw+WLt&zzCe~;GIR-&E|zg`Ls4plnlH*6MfzRA2Ao91pfFN2LOO{r`B`sn2ey()s2Bx>!bCX{{fCp*p z_jVu>PmtZ#(3r0^&(8f4^=>&;`PWI(TXdp)hzq@Rzvc!?*Oq=XpHo;Z};CxvYrMs(wd0s3f*OGe08aX?wKsBj(1S%OOU#aq9rT zWU4;Qym;2vq6U6M=bKp^o!rsU$;pZTTEU&4pC4pKI*;oYlMa+Y9A`BIBm{(Pd!r_s z8M4tYM37mw;+V5c%F4<>?g<_!!s-IpO8L(cTI$uuLBfuZ7{}28 zD=o5)Eo;#LDk1`1A9r_kC>{!vAkw0PP8pF`ln1$Dn)ZYz*~gvACvpksrQ?`EnoH>? z^n1^0TVrINz%f>eVJ%7w$OTzX9S=AND->4@*LYLPQ_RQbw^sBA;qZ!sdbiU{qX2xH+R6U8s{qEu%pUWZi-6qw$cY;gk;qS|> z*F}PE=F!Qe5wY&uc(IhCNbe3HrvXuei9wrUx#JBsQbSP7?$;VOH#geXM~ej#2r!7R ziArdzpZDi#Obg;-Y_0~#GrWq^Xy(vWTHQFhD>WAyY+(SvF(eNh0KN{80Dv$Nk9YkJ zOu22j6yeA;E}GNpYgh;AIqMwj%u^tp-tckdc=5i%U~%ea`Qdi#vc%%jVK9X2EA1|1 zywml;8ln(|e1f!`ckeUz`D1A`K-GwhymSOz_o0ze>YR?0R|gLW%vhuR_jHF0=h;a? z4cNXX<)4EZ-N|e42JN*^`B_|xaKujcdA?)QWweE!3iV~H9qy@|t?P1oKE26?35~*P zm|v}{Fzc}4MlUlrKKn9QB>xFvBjyddY~MsKMyrxjHJ1YryT0l?75t5`I}3Hz0i>>H zZtkveuzIIR!+I}3@+HC_o)LmK8ZEcK%5W@Je!MA*BJvzjR8$n!08h7oA{dJ{Ir+;> zxsKCzn&Sp*uADxQXWaQ!lrWs%eag5E>2k62_PT$w8FF;sH(4bX>(bG>ojM^Ju!zql&17k6aI(JUSE$@b6Ly8dcJXtk(QLazyM4O-w z{Xb#)oAW)tD^W#Co`U({ba8@e5%A>+eWBwAICu}pYU`(JEULtt0dXva_pp;~sbtpD zH%=fS4PD3FFfM$+#s#ntndS`GmtlKocP4Rm(Z9Y2(r_Q6|s$ zKIwKuQlV6iqwCAHXBnZAdlmd>>8R6vTAe|1`Myq2XwV1eV+_ll!Ss8sC`T)08aR?3 zv8cCt0sWJN3D4g@3}o;5+W*3<^RrQGwQ2ZC(FUgFOyGW<&aG6aNqI?-u@m5>HiS@B z5R6%B28OeK*xt_H5Ha+6e$@QSOEt!82ygO;Xl-NjC_^cvRu>%0Ym>mr+##Ju#}t$Y zO9}^zrLe?%#+{ie+MT#}0dyZ_()awba>JmExK$8Sz)HGNTOnita`LuSeATqQ9Isel z-$VU0yL?8C=+1)0WBx2Yo=q+DcblpeY|sXOS1(v z$~M|)a%~TzgVuawOJF6Gz)vUE%_a@viDWo?<9=7}7=}qb{pke|8q_YI#h`yBXz6aJ zw_7Mofu#b{8q*9_W8G^n&cyK%kMEtAYnclwH`R`s6y{7 zC9{}N`~ZyvG<@vxA+?onSsqP)hHBNgG%=nic4j_VjG)%%RU2e?TR>5;RDr{HsqN?> zzLy`GzVKT(+V31`YH-@+3WEt*MNb6MolXNSr2S20ZEbCGlv-~ED>yv2+>)(iRN@88 z0(1vbb0>754(pO{f6z4W6$Q*#V!FM3{hZDXq2^ruNu{$FiGZB1DC{#!=hWqAYx1>M z(r-At$NZUJ*^~>o)=CrbyrtC-yvFYp{?x50E^vw?6C@LvF&Qv|x3*Cs&+c%N*xcm4 z?x0Yj`0IN-HuWP@e0JUtLRhZWg7hB>(^-T+t%ZCngETwSd5Pzq+i{hTrFfeCmvb~I zmXIcNg=He%W{tdtMos_RU8gs!( zuVHYJvAW3l%cPTdTpjlAsXyMTm1_$CzaUu~w>t6Dc1Yu;RC<(eX(6D@{g;Tv@8bv%rng9Pf*yh60yn@Bg3X$@j}^5Qy|?|gN$(4e%N^y9 z<>oZVpCntJjJ8BybAI{0(4|=IXJx0CRvGw%ejJ%A;(;sy*#&+di&B?_bSSpF!xqL5 z?}ozZ)+sk$h`gF6>L_YFm7Fx>Dm3bnx?F{*e@)^W`*bAsgQy`SMy{#LD~TLWGUunw z_^w*p_kdk85k)>cz;)7_=PUbXM#><~aNfcnYscf{3@^(8uHuB^h)mZaLqzbx;0Ce= zGqrkoJfp23S^C)c81B(4|B*bY#ZcxpD?>RrAt4_-JdNE4)mJnZTxPJZBq+sbu}~w~ zsZn-u+kX8E1)Vx)V4P|EkX`K9G8=z-{`2{T>Y#7Rc)6@|NIWLhW1icO>u-~Wa{F_0 zqqtY2Wz8K0ag@FvenOidt5VpLW{XCUZTff;=R$BAjb~z?Z+*R%Gkyz1)dlf%czAdu z>-A*{H3M-df+)00MmC2J%qqzK(QHHa&wluolE1Qq@ zgq;0~RJsfD3=UlFlNc7AQauZv)tc}^5MhW5+Ym&OD>W7cb(@Z11o|WHKR8;#DBDNIoVJ`85u?OfGbRQVh+pLj3c79$h zZSmB;*g)MG&x>(0OsGqUC>LCe8>QR1Ight57%5TBe*ZL6gXxEQzXc)_(s`N{RJPjfqM7{s?#=fc>-v_(B7)OB^T}k8j*JYt z`r7Y>WHmG&B#*%C1i@I&fD)n5W#C+Clq4R zbm9qAVQhuJy16=ib-LL0JIDHDuKb@Q-#mIN%GKhxjF=@<+R zeP`V)JH8l7pzn*Nl(*U*#aF7mK+4#kOa`N;|B3q+yB$(85R79alSiE<8J);tHKEQl zU2cp6(P#{wQlt!y7QOIu9UX9h%7?z~;S=51OBMgd(s!H9!GYYs7k*-EEQ9{qN){Zb z)QkznH?G!I>w z_@WSMrLXR4wY9Z^I@Rp~zdobl*yELs`sV~PgCy5>SN)gYAA&ggh(h!#+Se%Ax|b(N zhgL2+oqeoYxKXS#&>Df0NsV zWIP*a;<6$Xp=hHh$!`ecT_4M;1^MRjz>X&A=c^}>#*229Nnr2c8@c!Ph#SOOG zx&Sb;Q6*TU*HCNb+uJWQm;erCEHZreU-o(-90zHsBfx=pM)z{cJ0w{8M(xHgDP%nPH?vh5Lzj*c$)m-tom zm36U7g`hkpRVr{s6mzhY|HMqlVda9iOYCQKZ62=keEEv#x?iqZ%!#sOUH$ zf|_*Bi~89m%em#~f=>uT*7sle{weT@XX3)Uq0PUF1XrX^~koGhP3Jyc8A_|?mtNxSP0!qN78r9$bojmuZfyID3>-I(>M^&gX z!Ya5?a)5-tZ+LqqViz7;`n}u;@>g-Z2BupEoET{OIK6MgH38+E0e&o|##1!4K!=sy z2pk(Y5`YCSnCUure=AuM!EuO8t-(Rscl%dDov078n;=i!IYB8LFIE)Cm@uj#7w4OQ z&Htk8t)uE{zHPxkAZP*ycR9Gb1$TFMw?J@rcXtTx?j*qpF2UX1L(t&Tn|$Bz-uwD> z_jvtJ#t0nh?7gb?uC?Zxa~1%9ref_mVYNZ>+U;$KCY9plDuO1zok~Bq|Drtq=gbn`~dG?@wE>@>jYaoY$;>jSxt$rH; z0IvARuPrQ3Ci{<+;gJTA#qzl#s=g~N4$D{E8IinB=XOEKP0(QB(@!cRRU6Ah&$Kc` z+>nKKT+1y5HbX9FP9F?{X;2Lc!#ucPq~v;~6;)UyvupA-4~NxzMzc&Io23l~2w@aE zcLi=!d=LmkzvPMatSvO2FlbsD8KGd%E&IVBHTdMjv>ZO)KM;P0VIN)nV1M{Zf(}@N zO&Vi03WniJ$wWP4P==ZxTUTj)vn4CX8pe-+Iv4@)BT1>$3tiNPZuLGw!ZQTxw=CcN zPDrly;@DX+B#+~Ul1M+AGDpc^uk=5d=(PLulYORAk6`t|o2W_;4vE0%_WNR3bIuWC>cQMd1A39H|>R z2wl(Ep~v*TPBl((Mu5Bs>rp`YlNJHI^XonDj9H6nQnel;W{fU~L0WFGK9ShGr&q5! zzFQe)EGCwoO=;54x2clCVmBiPhzA~Yy7R6S z+n)wyY128dM{zp+=d#uZnkE|M>r;I@(e;1ceIR+~+J`wEx_w-9%;bCCl623f8}a=X z=GOE{hqY20{VBv7j9wqh96k?+NFM~Z<0p`Hoos(8Rk)Orfu~}4p_#dYr|?xxGIPn5 z0Pcp+Y(c+1XtfQ+e<^?i0aYQii)FJYF3YS`_~;Mo>%LQLXLuT1GmaDk5RZspplE4- z+FQ&^Rcf@q$Nl;{MLgp8K@OKTn_H5et>3O)KrH83md{or|jRE-)j z^O0VL6GzXf-?a_?Vgr2BCl!I#Z#aXzM$4|h#haP?l(m~pcFC`fXVp?9<6m^YM?`Di z7+iVs+VQjNy)&oq{F%;#bE3`B3(#HC`@ekoQj{`6AtR6j^()hQqETJ;P>4@XWhczj_{9(lp9VFm~A(9*RTc`_>ropd$X<{BQe5AT{!m!Hr{p@6^&(h6C9D1ck zj~;C7cLd=4No<#+qoyR;^TtYrb4EF7xKoin*rH?AlsKzzXQzuBD#sW_R6rk;oM>cz zxt%=MI+~smXCqLtr==i#!8Jwmu|nxC3hp4suOam9(g)ih7GM4T#hz4T2EAK^rLMsQ z5=hLKW_Wnge>pC!zcoq3P3_5F_e4MT*K6-UM^CI59cb(={8@t)ny7~UGLehAX(Ocpx?7lo&w%H{A#Xq5~$hxn$?)%;mrq}ru=Y2L=& zINhf@)o=5{59Wm@zWt=}$-}%($NsT&4U4xeG^*fYs9QAQb_ed6xBdf1w~g`WeHPZk z4<09rY0Bx_pJ)P_SO7oG1FWFP>6Y8|uZF2ukf_A71uZx-iB9yK`8{|A0l0VvkzBnK zsVPV{$KN*&wiqY#O91C+B_>L&2)@X`ET!I>L5#CMELvJXX-c{AlNzpCN=jO*T~#HE zCM}$jta`N!6Pkl_f;2kS^Fp}e6xq`*jb+yR3B9T+?rTYg@Ex~;`K~gJ>s2+)y$wYc zM&C*KTwe1h&u@tj%>;a2Q9rjFXWYkn|C^u%>{&Y^hd89e1!Td2`YF<{9Jdr}*f3Lh zS`YHL(YUkE_@Gja4pNMtIH3UBCa7JVcfozGZxBv@3uycmI_w^8h2f}<j?^yrjM8E>} zgDXUtaB?T6QpcvFAniLYgf;n--<9y*KP(K^&C|1_xW)V7s~x2Q48Yu+;WXxQzrUga zdMg`ydnu-gg#12HKqCzb3M#Cf|5~$n_e;N|!R-e9!E(j>fSRE2X6w|J8mYt4bMJ~G zwunk~e%}}9LB)`EyTZ+_UzBwEJEn0WZ20bcQU7#DH+B!-jdur|;4Gzzp1vi?D~c*X zdJd!M9?6?HEK(w~4dX9dXh?0jEw8Tf#4wn-xFF?(oFkNRnpa8f9BvC2~QPaK0W$(Saq_X8(@9^f9%O=%T z<7Wo6Uov0VWw(ccuG&Pd;M+1c)1>sIDx1|7$>4gQr+c8F5&}8}_CQ4tl~B_E1hhyO z0KxuI6FCJF6H^+$ZwI+1XH^XB!s1dca4cpGm5$VTp&gabLy5#DxUR9$uk}l$gET#j z3cdB`O4Ia+?xNlbg2STHBB|RF#qUWE*DhbDX!7&zOfhR#_DTLtiC)QwPJ#lcMbe7{ zO|*#-RM|-kx|Dh!^6~EftV=^BCQukDzrYZB`2pR^5uZseDxyMCoi;bnhYugH*ghq- zi4QOS2!#1F_Y#guVbtXZfio|T(E^BZwi4y}h8RzQJpT-6-fjY2kop`B+ch+XNm<2S z=AUQG7YCCC8top=YLd+D;_$k1XoE5cJGlQd-~j?MPKZXQ1D_Nj&DqQ#kEnGIZ|qIy z%Vf#z8O!8YJdYI;WTR^Fd07Z^Yb?&MpSA6YJe@OPO=Denh>BM*#b(h zen9Ge1_ac2u2OTM&KL%uDiJ5w0Y(wVK*W`>E7YoR;4$f=0AL1FydjCY8FcVOTYx=xAKbfWj+`bE~yUSmTEDAXT z(YT>XMN&f+T55Oa1BBbaC&Qwn`+(}R*6C1*EV9xQ_!Fh@3G^_5*untrz7yrDG`kpt|C9{>P(WQFA6!qxPp`cvzB{1K z&}b9tYP(H;f3DeR2JHLy zqJRrVVYkq^dW~Kmz}6F#im0flP}u($~ffQiQ?lOnLhCOgqz zZnDDc?&&GYFCit0B%}(QNo{dH4xqK0uhvcc^Le!e9RZUbN^`t){pNH%)m!yq?Yq5j ze>_+cgHwt0e{z%KFoCQUhu04A(`-d}4<^!PtZ`?7(QE`3he=zb2d~!|a*pgXo_(FY zCcFHq(^g5E9c^-^siqf;Eo^ClQ#_<-gi6E^>4S^XN73_P`lc)iffPE?#d?hWuVB(J zMtP`2R3)#$siHxE^kBEy4Sn@{Rht^)WT`0;K0Y}aoic7qK#dV)36l*_#FZV0h>C6k zrUFFLuk@rrW)UGZf_jnoz9=+ir8zX>NQGvH-9qqmp(N_Wb2*?712Z!#r4$t^$K?3j zm#GwgbUGkH$5pDQSam;{fcSHMN`NYB{X2R@&z zyo=C+Nlyg^9-k{LlKZH*KS)RHSd)sA-yII&#s1)u(Hi}kD0ymKXkh6xRJnI*l(Apg zu(!~t19^P?9ZvEP+cAd(I2&JlNp@?v)@G(nZxE@j&kx$<=wWj2HL*muTCaT@^XM(I zH%|N9)XgPi47skM}wggMzAUJcQ=3cX>rp z>2U8A{r%*sieS^wl$H3#&&z5A+yDmx zSfquDG7^Tw#L!gS@_~i53izZmdiNXeU|>q9{m|qwSX7C3rG^T66CrO8CP@|KL^JxydO7%(1O)EZgK|pxP+)g@ z3%1)k9r^CEhp}~9}UEhx|0#?-Ou@=r^t%d6o zkTs5a*czg#>EKKg1YLo#v9T6->oiqTy52}IRf4{(1uSclMxZVAG2rN$Nb0_RO&JE! z9ZEAa@<;N#OHdM(Mz?GxJc$KMBXg~BI_^cS)QC!hoT(afB2t7CmR*4aK>9{yb!{3& zOsar-hEI!JQi_x+1Rzy@cRkIf4NYQ{lE*6@ZG4HOb&i;T(P?^VT#w3((&_Zk^>8lq zwrw^LMxZm! zYB`GqbUs+3IaoNPsghpy(ge(;FsZcHBjS{Ur7h#~-Qt^~ggo3S!F*_jDX^%b3P}w2 z@)Djcdl3doLSuAE-eM+2cblPvjOJr_0R2ypTC=e*rVS%uHZ0{Vz5u>HAf8el+xqk8 zqI^+cTy&FUusLpHP>p`4Pi|3QQb@{Xyih)8mcj+iKvWm?O;UjpgRXLOV`s!lPvVq# z3I_rLf+ivr?^QK@y>$_0d8h4qheHmA<^m-ho#Nx~)>9KDEp|!MXBoc&P&2SR=kP=gH1%&` z?y!Ut-`7VQrjpp5tjNw-YQO&=6o?&dkmP`LDQ%TK<(h`urIm!B^ zU@S$ zl%&xyJ3n0Rn$JKTlsN+eMEb+f`MVQc<)#+Jb`wy+ZbWZ@C@&=RC~a?IH7Ocvg9^jj5-GB{Zn|A!kx!%m zL-Rc>-r_a?ZOWNhu=HEA!^8x((cMKEF=$KXlH+z7XEtjkUytYj%JN5w z+Pjibr62bXKlwDjhNi-Z$r1$Xvs(|0PSmmj%TYE*?h)-YnvlFoc}J(fa@MS-DZEmd zwDZ^cyHoJm#p6=473Gc|Y#uRH=OpFS4@I6R!Z~%VuLl{!v6ypo38ZxipGNe*fb3}? zmpN<(ejPSUUX7Bc_nQ~L8ScJ~)q-MnC=`T8p^_e%p(cJ1QTdQ8zH3Giaj{E@B8O>c z;es7?!lzkKET8kd6zMr6R6xmC?hW1*T6|lKBfr065|DCQr0z5bSxCvILP)1N6a2lk zb;LB_N{+rh|MSXsMQXvq-1u%Wn~eq(rMSVTwkh`6#u-f2VCiP@&+4D~HEYmfxLR9=xEN_)71NgHM)_Svm50m?s+o~iFk3|I2>`)V5(B96UdfIaw(FY7e z7d`h!wh~fYTaZb>82`bUPwYHbW6-7hat*!{&2VZAnpo`9q*pCm4>Om`5SeJhzK@NSIG5U>q6+zT>rlNZJE zAWtO>MkSH!{)xR#XtG(=Qtz|pN=LT4po~mgE@5dNnbZXR6A%k2zm&&J^cWT?75E8O zni6$lWP4tzmSHtlOY(E_-@Q3N7%=>mOr;Gk%oA&!*1nahHGV9IRvU<-VqvtHA&PU@ z-T6hzi|g{~p1sn#a-#mTw3F2=epA4wZ66uhN605eNjGd2S#gY7`F-zzQU1?@qX|un zrOQNht`%=Tag>s!;C3vFsC)R_dY6#KRnsG8eOUQu>{TA7OiS0qcJ(y?RUlhx(IiXn?iU-C zSbpK;1Ab&-Wl<1n_|kBUg{B*h6nH?cuafXHo1uHc4yuv zRd(~=J9f(rIr38zNQTRAVsBK7lsVHX0f(NKRM$-vVbIAHzK6IF(`b1-9_vqHmfTbl zaehe(E`ruUX!j6}{q)fa_n3C1@WP@(RN(wg(E+Qt(ys&2(L=I^+pEJ{Er3-xB^rTD zCEBtykgQ2l>eZJo29Na${k;>*%}q!58gUW(mDTVa^BJj|!4?y~U-_+-=^969X3PWu zy_V7-Ms*t@lJRFjC%4Llkw17gdQ@ZnQB8#ItG}0xEc4_iyKy*rMwg*6WB6gB1HX({ z5VqrB_G16-Q!FAAz1I6hRi3xjdf0|PM`-y;*}W=D<$ea@HY$9*Id*=uYdg)SaC>vL!~<{I#}hclirkdRN(Pn z)zpqOM{MY;U@UN zsoiqjeJtEfv+unB;obhPoh8Z`R)-2N`qclzb_V^rp19AABr|p2xpGSV%XpP)#YcGM zu4MbWH+0Ii(fl;g$LOyao-|>1)KVtzN}*RK(EW}34~&9E`!A)FZQ&YQx`&{jHy|s~ z18PzfO^lQ z1t|330HEq8du$mwBaMZ>`PORG-)8n+0AZw?IgapO5Dy6R*~X8zKLJ~ftL+F;ekg)I ziteQ`DgI!!J+Hov#w%w8O$pkc2T_ctu;>1>( z08|N}Hn&%kZ0z`EHBVm&bjO?Ufb;dV-ERS>OE3=)9^k>l>2wgM(k7S52>~!gD|K5j z-o8WZN@Frs8d15~`3Ep+l38nWcLanJ3agdmsbjNaZzthET?>F0;*iT{M{R5vEbak@O+fsu<30n`1uq~DGvN{r zL5w%;?d!|a{$|5m4Zuel4QB7@bz6eu`R;EV%&K;Z!8KqNS@#`y zyt@Pu62=0qGx8}M4m%Q>?=%OF=Bj9xg*!yttNQXy5twXtpwO<*dUp=D zgGC1k-g+Q)kDf8|$T_Lk4;}Ad4qe>;KM^Ctd_Q?O%V+b;homn|7geQ&1Fe4v85t-l zw!*MDrLtf4jo-eds*!v?9-`;>CI^hVKGbuTY0X;L&Mw<_Mgy*;Ltu}a-*l-T0E;Dk zpO~zyEaidtVLLm!H@p≧K??C{!l4?SE3D?zdz4O|k)e&k#WTZh`Ww5JgmtUFjng zLvmiZ1;8c~*BS=M5ey@_Pez55Q}IEx`aV8BZa1s0f}W##QaGqv0H|xCo8C?>IPF$W z;+;K&t3-fE`F^7BJld$rj%UXe5B?dtkB7Qz^#{zJSxO7#QsFSb(P>0eu0urONI`?d zrqX3q-WENe3iv2B{{UY70Ma-_LDop~>wweYVbxw&b?_FG$pw(ru&5}R>9-yc^78en zzB)ycaew+ED{a4wtpd^;|A6Y9Dy^(Dv-OY`YPoHP-+e2Af$hd2cX*s#bh+%7$>yvja)i6k-W&yT9qx>6Ze1OMmL$*Tfh&R>wjDfn5N`+-Sy`J$0h(4k2< z@MdOamet8_yAoq`L7C{A&9H^`)WlED5$x0`aT6DU^-k{zc(812sOW$7baw+V^<5Iw zmaSZhj~)PGP#a4Tv$Ipl=|~vn?}ipiOfvTYhP>30dfZCfN$iuOS0@TdJVpiem~_$Z zwLlRrd9@+F>yzQZ8eQ`9@RFAAUND2%+1`*Ct+It|dGmht@;iz<&P&jvzxhy2WkE}6 z585zM2}vVNObS|kFwIF=v3ko<*zy_}5!Emr_5%Q83MeT-Y2hj;2K9g^jH6S)DyaUy zS3#TRpbnJ0NZ2dUHnAWo@I+!jq7A@Bp~+!=VI8yRSWbb4 zj)M6!Bw!LJYkcVOc*_I`Vcyj{O`?#>23=>vk`?-qyGL=IIFq%@leYu-9vcNY>{hy`;vi==A0wHzGa5rO@8GrZk z=6&4$;R}zq82?+pS9)flX|Pl$mHt(GlDFVR@jKtjlS;!=>k7e`s6-`PsrzII_g`Pw ztWs7wO$-f*>GZGea(QDLo#+7*5m~qANB0b?*@wm}c}hlJ(P%Sp9G4v2VGR{uUl5s` z1YidJwhZ6n7TmDiwjL zOEHb>vzjSY%ry-zk8xtAR;LO~LAk%Z9emQS(*9O#WMjqvS zb@6reN>Dv{ZEH&%G#4o#ulF%bKQU1e)I`z9kYdA$JLvmkZZV9Vh%RF|I9t;?FZ)iJpzU3fhtw z(g>D37Mg43Io{-1igCN4Xh`=_6WJv|42DBdO%Bh`&iqPBN>q3Npl7$9*MUIrp>9ys zvrMUIM~`N*`bFecJ!l%7iMB}yiixn(2nqyG*p_&MB^1#V%M(+ur`VvCtPdod<53aH zb5fuaW~y~@0TaFin{CtH5q>F6C&~RnQpKFX1hL)MR?p|9#`xbmhrW5JeSei+51K>s%+z_fokzUma!ABGz zg2=Nku!REgZNx%xu)?AXG~KYE>IggGptRI|!z~G^Idw2ChCk4vjUQIoL9iDZ0g)ub z^8Le^@_UAxh?Q2|jbtC3-t)wG0@c={oIbnX%lM)fS6C{n=efl{LXz|&_58t|k0ZT4 z|F!f0BPLJa|KamT|7#a_&pTL}8QsHQ0Wtix?z>}cX3XY)(!um#X2=I51nA=}E^PgJ5oDcA-CCbZt!AE6Uvs%U$J6WYE?GWG7W6wJfti*T zH1d5w_D(4jF$?Gwi^K(%v_&yCGuw`(sN}OqG%JQv=)PnmWopFjM)BZG0TbmT7{ff6 z2Ypgi4?w+^Ko1eIo(FJ!rkZuQALBlipukS@YeS}fs2F^^nZ@mD(&_V@jzJ+vZIqXn z$7#3m#^v`)IMBV3K+phWZ>Czq>j0%OZZQ1?mw;TzPDn(c$- zMG^^|aG=du-yO8$1w}G{PV$P(OR9gXCs+uN1i1%^+r!4^cfE6yIN`C`U3GEo&(y#P zgj6cxH<=>0Cw6r_bSOR@5O&V?ebjaGBXe^YdwUoQwH`F!0-K)M=D0_}YP8LNn6D@g zjn&c!(g5{!m@bK4^BvIZr?^=4kF8B9gXbFObl#|}wF&O(S$+H4rhME?RoTqf=Ce5! zwpkAIFp{B?OQ7hil11c2gLTriKmeQyQ83gl(t<^QGNm+c(HH<#}RA7 zKUBN8C7mr0;fO`Vmpde_B@$6-V8d~VQsOAk7HAvY<+^7utD-JyEDxqKSM=}htV}JT z3>nGKE7$c-2u0pFPXZ5-3B_TR%wm!HO-HR7x&jAXi7~*j;e?~~Y-tJGo4)wLd)QJZ zsxyy9ED(ewf<+-z30s&ei|W6_03wfM)Cw{M#}!4=>bZ)1r|BjKN$3^g-7@?$BS87I zv11k+3KxqNl=4bt0{BDRF|XC|zglN-u13osFHi7Qy{ZVMyuMK1ofybs^7K6jEuC0S zVlm$|w6V4&2v!i>8!0fxhNf6)IL!86edrx+Ncv!W&;`F~g;PgguJZuC?g_~E6e4p~ znTKSruq!Anlv0(1h(eR=mGdMcKnEF7Bwf`DR?LSjhfhsl(8{Fb*31cabWkoVx1%}) z-kJZfFa6+PA^#z@{%*o-WyTwY{mW?8^%(oYlIQC58nOR?27X3|!&Mi53A~6wU46n= z89c8!24e7=W^UI%4q`GUwD9iDSW*p9)%f(mS;!aqV>y>7zCh)T^cUU`umNqnFgA;( zjF>{c6Y1*w;rgj*um8D&Qr9&=%aRKOT=x>{8P#w9R?v;+;S^+1pmfCBsGu~MZtRe~ zhx#yp>G}_qnB&!g_={7s^?y2~n#hzWJK=!c4Y>5vB(2hU)h}4IYe(Uo=mR_|T}8u5 z*m6y}R_fc1c+i5^mKBK=6UY1vFox+7$Jfn)*?VL$0c?M~UXTN|-SYXp!o zx!gcx)G!t*i1rl!a8BRlxLI$ze`>M^EL{KvK;hr3o=EX4j$v)AioN`A?nRP6$=d%_2>7aVD$Cj7{6~C zMjC)BYvnsywyu(Uj+?rL2K+0?3M|vO|aqY_*aHAG!5qPziUZ=1=4v* zy?>M@{*(46RvKR4LKg=|fkiDc`%mlcq$|E@h9yHNE|*eJpAgpmfuFnqi!^Ed!e)a1 zp2=FrdckQuHjob+iaCkO?|vIAC&pA)!syLc~Yb;Nu6-p zX}u8{K zLyxB!^;r66M|3oae|x_cD-9tX<@+9gAB{2~NK&FdUh%zKj@uYwAaDk6ps3+tkkB;# zd3L&30ZzLq&h}vYe?XtV?q(Blul_YwX@Q9>{+YlH(wMJsV*b}V;+5$KLd9Bk%hsyO zX>er)^wwo+T5`Qm%p_egtbffz|C>Pd@3{ELK$u{Vn_&sfGxif-&-CgA+u86eJPIlD9&GXcd`W<}Xh?Z7!SF z)0=M``CU6}Zno`z|IFfAZrjV`+V>vwWP%jyg2jR$F#uk=AVx^BTzCi;UDd(cme>FM zufIWnO*=e-3>)kIpP&Ex5rtsC^lEAouhswUHxtlMKb@Q*M9dBU9sj?FEEVdKQAcO) zu=(G9a{vix*3$AOzv2IW30<%%#0L9=e=iS<2og9~W8;mq>tsPpSMz@Mm(NYJY6Z_& z#;U$Ihdakv`dp(!e`)==+&eLA+yiT`RM@vuJ=X84?9)9f{EFTLa?C+X#JpMSCKa*UfK ziD~y~{pH!U|4spuu}kGot)hLsrQ=1_Xp7KvftHv!}m7H0eO zZ2Qe_W|V+F$n796v>uorZt`21i^$C=mazj5U7PQlA-^&m$EZec00W?L_7%IHDbMz{ z+sm969&M*dQFEQ%n~s}y>$~059DBZs>XuI~&v*OtdzaVWI(a<@xYt1TX$CqTheb{2 zt?mt+i;Y}&HsIqnvG))+!JFR3v;kaLfj(a$Jl;OjdGgX{9OA5<##os_eF8}k7S?#} z(k#kytg@0>x1M}$vgB^Q+a`SB=dfK7jAJTg|Qa6Vo za5E6=u&S&R`_-%CJgd40z<1eo%YTYF+RNh2&v$ZO9?ie@eBd^Tr2DFIosZd0X^aVf zf7AJr)xOVrI;5sy7;FtZ2qV`3;v`mo>b1|k2q2!>bO7wPNTYu@{OjfU^kqv=_31ls znv}U%`<#|l)vv~BxyV~n{!juBi?0{MGy{%!US~h*9He-rYQAqpEKW%?WVP?o*$x3$ z?kjo$oNUCpY*o`#EXMRV|}g)qTqD%BN;k4vl*V`2V%W;NU%?f;9a)>^Yrzn-;~>`H6E&MUjX&b zN7Kp4$;8nZ1IK(OkuGmTME!EYAy_9r0sanXvzCu;Omaq0A#e~_sSIA52+?s^@%uoe zH;Hb5`;I;G*~VQTByijC|Ft;o^UT@hl2Qn@a1;kSI5Kek&dSm zTl3DxU4{nW;4G#ufH0{ZKo$74O_NoPj`771`wOFdZd%ua^=2d~ve52zfuMRn^8fz6 z4IY>Gy`TO8OW^sdZV@=*Dv1|3plY#Fvw;~ayvFBG#Uk*$-26h#xfL;a-=Lw!kJYd1 zvPcFT^7SWRi0k23!$^4o@B>rqU7zR693=9?G~<{ddf=QV11L$PhQ7-mh~9_N`Vl<9 zH^uzw$F=>*_pmr_7D-z#@%6Ohq2Kw!Z!_q8Q$jEN(7gwDE$)Xkfy;tn{B@1*lT~5! z&6;PRpltH?i>=S)m#c5~LFv$!)GOg`M`a~ugBiY$+k{B{2vJ=g#WS%GX2s4<9v)n&`rEE)|U7N9W-%KKa-23jt8 z->eGT4lj!a|Hlg;LN}YHrj0o;-KrYX*`pgZ8##*bqZ*^jab<%XBTN!RVy|x+xl5YQ zd8EF{FO|9xsD`&BBKW^sj;ojMdC$Y;e`a51?we5LcsX>vX@KmRmie~gARRJ>yBTfk zW&BHr_fMJpC?D{T>FY`fj3<4s@pLHf8?kdR;7`^X#4`d+A+UaBe!MA0xSy@@LI3#+ zsFyQ!U00pYegDj=kF&42Z7+;QO5i^xZOX9&tJIan<;36&HdV(#5~ewog6ad~lLDB& zCb$O2{|CuN2ji{l1`d|Lp+IixI^?O-`yN~u{3rov$ef3j{d&5u3*hdT2j7<(Rtzx~ z11Kd4BX7nJVGrKN+UxNB@5m9v{P9U)>x8i0L$)$9@+0T_FzO(4^C9s++GS$;4j(n` zWmvICY1zqjgu(G0O3DvkuX&zY`$J>E8sq-55R|5^6FCUO_kBEHOpvBe|A7pS&`{@h zDK2#dW3@1Eae>rLFr(WRy@h{)-XXlBnw-I&gZH3U3P*zz8*GMedAtxEbl7~ zE`S5l#t=b5TSbosVTQkkVJkwaN)cUYsV{zF9Y{dEU6!T(z}T(^{%zKtl?pmw2VRb= zVl#Kx{*xlP>g$j!x&?~_I>&H=ep%@Np_sp$I24chq{PfTNSE#^Z6VL-8@1+xFh9a1 zXZNhS{%UP0Ov7Bn@DX%iGHidim)=7m2__N~@9^{T`tzJbz#wsmL@u@C%|wt@G)#%P8Bt44mbBgl>EZ?}_Ci4FBzq zg^q+dzOa^Y~i;6Mgo`3h5%eQ+RuVZ^{n@e}-1jroq)Kn(e=Ck&=l zq1RB|V%2$mGzEAmFfRO%!{ci==tBNe@Ej&|;U8F6RzWDnXOOfIzlqJHsxPKNEL4eX zG>_nT7RFqEDeB=8TZ*~2j5@87f;7**>NJh$&3Po<6$iT&%~%&Em2V7v8BC8g4t-uZ zgY=dRZ?gF~sS6Zx&ft3|?E>spUs)yBExFpU3_eyxPGkEE0vkG`#Fg$>5EcQ{BOO+@ z>t+U)za(eIH@sm#i^iZ_cuZl^aPBpnAX zlk6<7$r1gEmDT=7iXjJN3k|S+#Nj#STMgY&hS_=*d)`=nGVDFlC>4L;L*Kq(S(M&5 zXiz1(KHTL=9^;gwJ+@xn@xJk)IVpSyNC5eW1xkeZOVJIngbCZ#Ce54(Nl-P8jlYGg z*1EtRhaz)pnD)vciLp=E8gbBq4i_}Lt$Cb)@QF5aI+=w=tuj5Tp7Sf}J|UkAiz8nC zRt=KF*b4o1#5PbsAND&>eD*`p{#F+0`CwAaDuYYI_ZR~0cw|!%bwrxDER?a7ACB%P zbEEt(kBc;V=MjK1AhlY(*K?w4lG&41GvfSlUD_($sF9=*h(>ljp?6EAj{X;H$dl{f zdL{BI-y}l?m#S*QrfKT@dsBzVY4Lk zv4t=}zt5cd(Pj{91!((erXI}YpbqvWmS#W+Z_$6^s6Pn~;`CV}$OTjs~X8i}ITDAK{$x^mwVmDbPD; zeq+<~+M@;f9#Nme<_v85-#=zHeJj&9w+Kw-&Fx5KS@JwXe<1Z@HVI?AeU#R}pW3r{ z{+q6+Q9_k_sTlFVpP%k%1qRFM6x1RQfHl)&rTl0vHz~^km2x6YJ7~_og-P%uq|1zH zuF#NF?tt0l+D^VM(_;{P*=7<07t0w~{~%JiXON7Wx;pq^443wu*2w4OVcm9&UF7&j zNJ~a>r~rq^`%x!ZB*y{v*9`v(9Rg2!TLjw}r_ydKuyU~Lx`~WXxKm0bfY29o+`Ik%`*1LwD-&87@fCeyf)JkhQ}jJl7IdNYubLKE)9< zW~**??*dtgoJjch4WY;({DCBw{fhkDazM%#Ol#3`tb02Ad>-ptD6JfGK<3hlr2Ga| zqSEODZr##&OsFL*i1HG8@sU)Ok2?GtkXQFre~>}yK+pR~7{{nPv=6qAA?4WI@agV$=hHLZZU?Q zXLW>%h?;kL)Xq4};JP0B0gDI)fv8ZMKi4MZ(hCxou`gbZ9zKUKx9i%+yx3YLQX{a> zSb1;M#QWZ1br8w)w=_e0BRh?V+ya+h$HVHWoHR!@yYV_JBpWPIOK%YJ4GgftQ*oFQ zDW}!6Kchb%;&4dGl{?8B>}bG=d!h68JzX!?#N)h+%_4CkHZ;>HdN~m1$El`PxG;xY zk)h(h5}VW~?qs#>i-z7(zzPu@NZyf($_j* z^E;lEJs^jI7emtbsx02liTT)-`{RvG{Sdh$S~&eyDo|*gq2At&`|>@`#&h)MgJhON zaBb4Oe;E@Zytv;s7zZFUkbXnmBt#1Q2x!Ud3d^Pj!oLI)_R?#jEgD7A!K-_JlzI)D zb+vDDtW-ZG#vct*zw<3mAHb|&{@$|Zr=baF=}5x9 zmOHO|Sf_3`*-o^QS9FL3iZQt&)?g&}=b!dUiboxgeEpI#SgMyOHNjRVaq9D}Qv#1y z^=i2n;QsfycXo&Nr3;2H?zO-gZ}LQcu3`0$e7Fsng8ArWJH+aKJ27P=SM5(RFXh@h z*_C3om0Z&}B?Thwj(4m|-$~-6K&s*liLH1o-0~$Xx#oAMQ!ni48;YtXL=@XU16Hr;%=Uvr1;7fI~Uw=P$P+=qo#(zFwHT?NnWS_B=zb~Y32+vj1t4x-XV!HhT3(}knP|N^eeT+ zx{b=DijZ#gjJ=^?YBeZVx*s&Tj-?fc?kPYY7|ekPhclwi2~R_O1GSw@uE6Lu;}3zd z7cRID>_9Q$*&@_|i`xW=LgX6ZhY1?uGq{LpXNY^N%vOH%&C$aSZ!n>=$t*Q_DKEcQsb&2G^i>UQ#OJf=l!_itEKa` zpm81RSk?VR_ORFd=nQf?9YwC*zE?Jfe53Nq*AM9F^x^$t%lR_lvVBjzPA32^VIS)w zLi7QG{0Mkun9|C`F`#l_we47B`E>hMZHX_NZbDEq>(c5KJ|1B1_yVe0aUP$8SWdyv z0}4x}L0|I9CD{?OXS77cYIMo1(df2F=TML}d0O+d9Cy{7gZPVkcKyxbSd-*2VRt!Q z`LXzLZov&}mPi}-(yRRrpE&c$r@n5X%l8;$q#g5KT&eV?K~8}@L)!U)hf{ds7!;Z_ z5|LggqEYPZ;=K_F&WB}4krd9m>L$k{EFykiwQqEDSC>5uwR2#_nIQuObMX=6G{~gj zyJt|+F9BSQgF1=<`7wlOn5dZSiaPFY&0&OYo6iGH(h0_%E0e6?wnC6(I!1wVCxVNT zO|crkf>H9}w`(W&1-PT_>$oAdWMYmCw{>1<06lz6LK#9(M}H{)<0hxL*V{;A9)$Wc zt;rAiq+vLD+`$W|<7%dc@ON(6Fm-TSc0Tlfyy zDA0&=8}FQXg%T~i@u2c-l?mIQ87cv3u-@|kGevO{((s(GgC;WQ8Jri>Xc33*%{LeR zHg#7yHT#}%?@I`(f0_RFW@UkuDHJ(|zY8s=VLk>q8Fcs*+nJG_V0ub88Y+_$Qv@kf zXup0MyqDhwV#1}#5S()RC@L-r(~s^$(!`ID82JVZ5kcT~wdZ22=7}*iV+G0#2)SK~ zcMgH?*)4XWl~^7AFzS}+B`G&D_VHEuY1GCM%$GDI$QJeQ?dmF= zAWS52?0^tmY81N(x{gi%j75(jXwYwv&uH+&2u9`u`TnGFm&O~B<|H^pHR{xlxfDHH z@1iheh$0@V@I0+Txqp5xU>Mqh^)U2WwmXPWgpvVN8!k)T3-1s_MZ1ZuBedIoJHGZn zx^h7PUVgx_aS<)>9+IVLBMFOMtva0(U45RCUH1QC?=8ck{Mxo}It1w$KynZe5QY*7 z=>|bSQo2Dx1V%|kYG_BgySt>37`j1H0RicfhM}HwyzcA1ulxUef4BF;^HJ9Zoab8W zTrrlz_cf& zbX{d$Rwg05B!pq0PcMs~4vmF<9qGipA+llUfzf|o+1Cmi+TxPY=-t0g7C`pn3-z)C z2$*$=W_fX|*%vPiepbKM;hM2_ofOgy7ts*aLq zN)?CfH)l2kV0DTD;6n}OAK{uwE6R3litR8_4q3`2#>?IDh0W-5#Wu3w^TsInPnJFx5+rfoaO(v&C z-%p1GMhVG#s;3kC?WAGhl<#=7Xe4O;)k!m3>t%QKt4KcfS->|xz~fX)^N2O)d`p37OecN)q|IRVy!6mDP~{y&~KvGK2KO%ZT0w2_9OjspGM}ob{2-_ zmVOr&jqn47wyGSZ>a&#oO0}21O{4yPuJuS9n~fw{_kj0N(#VP39)YLCz;~rU=r3CG zT)srui`yyFJ*4&(W&E9%vmvu{Vz~OuKvberuUd_6(|~iY(7rvk7&o7e!o+<(+eY1f zw=Kgd`INWB;`y)59aM0T0d=)PGe!al40_*dE)5Ea*?#lYEa3Y;&Z1{wSl{>L;pTF9 zD9JtUe_TnrS24>_GGadpp2oI)xd6R#p)pz5o$Q)6>1mWeB_`fU*x%$VjOhGL()g&> z>66%uRp}4TvIMAWw-v~jB^A(Aj** z_T_a@B}4M98^ZJbp~&YK(paDcrS0%S7lHm>8`LoqVcqzuA7T^~XV#-u8e@T7mjof| z9-$SReqC0l8=>)t$XZKdK#tXv&;_^@5oUFkcMFG~UsvuEcO zx*K<_jK{(KBH9O~gTiK;Z!Eegeda=Pkp1Wrno#gykwxlivY0$9#rc{iO_hQHvSk;k}yc`u9`Z^3>6d4L(ZXlMH)RcIL zS-ey6O(GE@H(kXM{$5le>y7qDI2DM}ohGGd!M#C8>iqLtKJg%v!?grc2GQ`)MkW_` zP}Rh%-1Uu*{-5kS?{N@7L0|m+xyX>65C;-9{0O@EaC2^s1X~xjKhg&@0ivj+2&AI_ zJ(acj8nptEu?pfii!*g>DUdI4PNtsLMyu_dR=%?qRnj(>2PxpEiSKtMx>_-lzoU4u z;2lteL4d(yLGmz9Jc3~PEJ$9L>Zv)xEVt4j;*cqEYEnb;*-3Fs2fh1CzQuqkOA#HH z?{(Ddd%R-j+3p+msDkq*GX~k?A-pabdo)lyvVmbW2o3pQ)7;E4J83Kr@f<_J2}fRy zpRW+k4T5SByUQlNA=du3|63$ApTS?0F>aH%`nA6~jnShwzvqEnC*J%s_By^YUZOxE z1QaQn5utW#-nZ=|R~+mM*|vpwSJUdmJD--jx^?B_GG5QWK@>#lYB0j2bvt1LY~q zmGaZYPl1ffp*I3A;r1$FQ7CEnYehlIg((`xL zIp49D?5kO6lC}NmBM9AqN zLA$@QQ=>IRFo+a^@F7 zt{V3`8|9w7HFz+Pmji$O3%1kExUw0L&iJMsUBo^QksK}(%Aw;i`73Ds)msDYfrqB)uPaBo z+q*r0%WnmaaNQcrq!vn66tfv_#>&I?s(bquJdzH z7fh9oYBK({KTEdCciy)8NAycxZCACn$WK&!lVy*3p_6>-0@I%v?IGsn^5I4V?;Ecp za{W>InT+K!vhTSmN-2^T#y60d>Ixoy>~B}OFZTjkiL`vSGYNEduJ zl66vxTFd?Hpx-dP^F>f|H&f4X{v}Z;pu1p{Z~kQzuE5vBUUMiexfkxGX~J0m6W$Wd z(_OkZU|%2oG#%Jajio-S>pgiYY^tsK_{x`ZvO6U3gpl(KV{~*4H`^Ln9sy3cG~OpZ zV4!_!8+t-u9Ut*EXzftdT11GZZsgJMlONniL0j35I2F&%X2ERc5E}?+7=b2bY^M9v zb|qm&XX<{2VXUerAr4`oq32NDJV=f1RyFy?p+(o znDU(P+2zD^B4j+jh(MY}i1ewgdG1qlaAaxDTG7H$MnA_Abs`T^i3LeW{xzJZiQ_0I zgO!DcfcXV{i{)Tl^bg~bk551txeA*qnRk?U{!Tpy+3>Fr(YXIA2$JYJX&Osv;W&LO z12jHN^912m)`6p(iF8V$+D=zizZY~bVccBo#w9AfD}kjuYup@yBoiba3t`Y4Z~0h|0X`W4Y)b*`WMBiZu~3fMl{%2l)Mnq%dNS1!b3`! zg>(gG(IP`!8h5Lv664NYn?jcWVU@bl5TzQnkW6OE%LUFkQfNyd4cJ92J%w_$J67V% z#wDU`jBuUvl5)Mg$xd_FZ9?8_2HwFHii`kPTvP1@Hl#?C@nlx<#zW$h^a9zFcXFRo z>_rdCL>}LTuL|B>;*jCUyAcAhl+v&{V@s2KsN&jo zR_b3d_NItqZAxy2j-}Um(@J<{3f@iHeN z=ej2CkgCd@2#vz_WHnCP`aa_d#LgfItl<6x8_6-g$tmJk<6zQF5`R$wbsd8hGsxaz zWpadUk5v^eTcB})E>(CRowYIBbG0u_QT_EwhI#6jPhMe!Vby#{)+?QCEpBpUWQUV^ z+*`*Dn$Zn>`welpUP&U9=9SNH#_y?{{!Fu`uEf1{yU&VZvYo6ujQ;7?oKSYByRJcI zXEZ^~%SGmU!!L}lVZRu`FQl0E)Rrk8^eB2oC7qC4ZY0{3URyki{J7;%N?Cms5!A)6 za+&A71(0z;d*+%!P<6ju2h~8f0qd+!^0ZVRznJnsVVcvnd44HQU;811XtD|nyDc>W z!vjyObKhz;Zo$Z3QL*T&@dFA>$UNRJmuQzbptugmF+H%ht>(h3+c_KZ8u*YZh?vb+ zeZ@Hhfg?Brg?&DYOQRK{(Z@Q~&jH5+D4%BL$A5YV`sL|RFR zQoF@50%s~s22taAIQ~P+*zF==B-7>>CSg-nce&&^Bh8ON#oNJtcwVMtJK-BrNi^6N zc&k)zm1s7)87a=|-lh+RD64xB`j|=OmZyV=;IpQ(@NbEX_ij0?!LNA*=CdkTdiWKe zFb<9LzRtJs<9DJ*T5OW{Ran(=bMnUPTGNsR4AbE+sbSYC`RM6?rS=k+gE8P^z=Ex-S^Jwp*!HfSyOYU+FxQSb?1#>&<5$O*LmDM#9W4%eOo^(c5;73;Uy(Y! zHk|7P^ud+!LCff$+&en|z)o5|`E%Du1?4rojFRibz$A37wX7;`4_xnPj)`GkSh^45VS7c#Ho^k zEhn#FD2iP|lgmY1l)VE_+t+t)kW1GOxX;)I`&)}KovL@mK9^s5k4;Ypldq6t$cZ0Hw!d6m%~)_qkFYV+Q7QLUcE)jisuxkFykC2fhN#7 zCLX-;;0*aHawg5Lm9_a}3qLc0^?olZG3%CBu=kTyQbG$Cb#acXZ2!ulusf0bjnSrt zB^G+PcI_#sK(ljcSPu2Pv8~7dQAV(?QBk9Ig@09I1)jp_@HzfRucMv11@ksx%d^8;NQ4SCEIH};(}pH4_*4KWV)^}ZpeE?s-+X9@|z2`^r8ldZ8B9QqUcWWE;}iT*2j#TGye zyN=7G!%n+W<1YIGw^*W19&}-coiNDOW)K=&Bkm~uI5o*R_)~U^g{}%&iS$?I{pA&} zEvm?2E=o|J**)*!*Cyl+=s#7-_=mSXKYes;;5ntEw+5(1J(O%I78GCTUHee_I_S0% ztWXu*OK3VNWcg#X`(h=TMLAqh5_cQ7I*PX3>dlcS^L4QMZEvMQ_^) zv5|Hq(P8scRpi1PjcqDiIq<{4s_L%5&q1^PmtrP|Ee*!@e$?7zTN;!+ItO1)zBuwurCq4t5Db<-@+xA+yT6zQLWm$Z++kwIpf@rnyrjW!t101Is&0butxh}& zcjQ*V21*E?S`z~%NEShMZfdyktaI<@F+w8>U6P^>?hF4UIiFQt^_(+Tkz24zdU`Y5 z%^lPrB4%9izMG47hOeBJAVUN>6B|%fDtBa?F!;%qr{^((&ebrAL(Bv5RJZ0my{N4{ zHB21Rj_n8P7d`*!2L2G?L^-MXZs0j`DwlkcR-xGMPXHsLmA?*qg|eGqXA#?#TxVG; z!4IW5CG5lWz@EoYpAR4Hc(%f35K8(5p`3FihOK{Hl&%Wup-<9Q~!nF>*eo<{1YZpYRi^mwZ^`T^y z!1SWiQRGtJN-t2#%OErDi<++U>NoD6>Ww3ed{(YbfUO{U6^G7|r1xb#Cu5R4ZudpK z)j7T%FNr5>MoNQD$2+UI=mDj%lbG%HH9!^5@l&ZXr8~-$rfO1qqvi`eE6psY>U=|d zdi_+h(qXb|$4z8n*X)gI(vxq4fi$zf0R7{3;lMjPH`;aO7xcHo_J%jYlT-JYmxM87 zBR=Xn@%LMX6k;*_qTv`3*+`8ST9*yq>{L^FyE_NyNHkM|bRTtTXgKBOrx-Fj?$9KE z5xV8avzMLd-UlYpbPl``KM*1vVSUT^P&(kG)=Ep89m2lpuuc|4*UzQqmCHN1u^;nT?w`|NvFkR$1QY!kn~CC2UhjIdHUm2MC=rYCq*kL!k46BLCV zqmvq-w^jTi*#4T-jBKPF8@=F%2f~?vX!D`OG%VO}l)CLtkXg&Dbsl5o8#KPHc%Aw9 zRji13xha-dKI7j3>9L68P6OcyHbA@W@m({EE67ii#!_a%Y7gQbT__;d&TaORv{`cZ zY1-R?N1E?wFhjd_-Ak;H5|-vz79H?3A*#n@mT)SSFwf)NUGDymN}GZmwbBu3-R|6N zf9TD`ZO)Zef@vb~+@f^lB#L&M-f0dM+)IE?QXg|lC?0w(h=hv`NwTIV8NXi+qFG`q zB)V(5{;RD-2ftt+O#kgCQkdLIvmWH#+rwP$}ZKWntZqecih@;v}{kC7R5w@xQT(#E|jiGj{I)8T#eha>}B)YKrsEbG6F?Mzwk*RW% z9vvZ-dQP)r%umqY@C*s)bVtkGCVfKtcfkj9U(5G#8!8CU6Z7>NP)%HBPxa@2U(|aP^_Kf-tG{B-41P-#jO84z@hy^!*%*>&5m$Oq9wekTwIT4`Ioofz!-aK-bGItkFdu}$A6hx0^MZ89t$ z!{64{#Jo=6%$~X|jQy?p9k=FC-@r;-EihVu@9VNQAjlQpP@r?7s1M&N_R&3hMhZ@> zPFdAIDKa?c3BxaWQ=}U0T!;{(o6F1y@uLH>2P>f+WvBE54}velKx^#*o`j;tFDP$a z-Oh*)+V-1&9r%;*)A;+bybzwH5G&b!xjUG*VOTWqaPR@6b{QG2+ow^u-21P+S$r|P z5?>+QiP=G*z!&P>^)Ow$)vlr5l=f$#_k)!UQy+Kzk;=G0u&;o_!Ac;p?L-``GS26h zfA~1se^Dlt$Eb@w!T}F8t6){b^{gbWLZlA?f0YKu4v^ldGryM@&0%Atxo!OmOBSY_ zsoTCHI+AM6K!Zj;88pFX^Sa2z__|>$;}PB;=Wrl#=re%>|9ghnJK1`~YaaZ;s*F3e zMXTJMi8envslhce6x-iT&06-vHv&_NdmW;wT6?wMyaCv~UR_Uxc_-SkJtLaz2lLtf zcHPQL$hR^_8tRU4QvTqBKVKY1oQH=p$Jnq{`K%`DpQpV4`cJXU>xx8q9JQP-cR}*} zpZH~X+P~!|4|NV{qLXU7m;`y*;`5l*xfqj%x9r?d>Mc+^$6}Y#ed|F7)5Eu^mfJ01 ztbTKUjJrDtd)Ub+Cp}hbzW9Vm2<yw;Ye zU1&PM*z+s?WA|#I8;+{rP-7TghL$-G$QlSfia26~6HvS-6=_f9QY^Q(pkR?nRGLB~ zsor-?8B9<})Rk95n^1ZA1SlteHdpY0HRO(uXhM(r*+cZPvaub@!f6ISzglt|eMYBJ zFRvNZSJ_RR?<~}KuZk_(0YW$q%O>Nkp4?1glJLhUlbB~l0)8>EAO^hiS{0gy%stXf z_g5bLaBg;zWuU7F-Cz8e!Jdsy}0HoQnBfvz`~_){AtOBjsA{7t6PLkyIeQ@q-g>Qg4}` znp`uQwr%o;v5%*wdh| z`IViANOh>LI0qHDtHKhgS!GhS2~xLQU;_D)pc4)*;e`rO{sg6$=R8^uqI4Q+Q`!=# zH#|m{9wgkw29%r7=q#I3)o&+zG!=h7^Z-M!@V*nu-B`**(TR#8PPokSmmL;jHz>)2+ zW*ha2e~a$z!Ee>}S!`YE9T@U;FV}Shq-t{88!YPeXtO$Nmq7NBYtMxl8TxB&!>Z(L zC1dJu@FI${2I4Ci6fZ0i@jGeR6(_Rnh7FY-eYcPpFYpiL93g?I!dzc@zO2$XZ- zb{0lpQ&6?sZYSj~LbTW&Ch6ee4|5 zWgiO8eMeW5>x?(c&7aU2)ub%Fl7fgq&9!u6scdVqM$igTp8GXKuluq@=e!LS*^r0@*B|cDvU+e}y8zx|UjyX)fi3NvArD3yg2?uTI zZ5d*NyX8~{(wT|SR}qy`kCa} ze2etc)IA{v7S^;}nr~+lc{D`6Hg~EU^jF7PEea&IqQtkl9eGAeB+R9T{RJ=B06a56 z&2(b7^-KxRZEP44h)XcB%pzv9;w5x^$Y+H@%q()L$;7&LKZECSEpe)pVxTpqXVnSV zvXsInT`a`YzcOn^M7f~d=IN005uO@C7euQCgOShIflV6A`uLVwEs_9n8O1pM(dxi~G;qRu>XK<*^WbwY%z zM|i0O&spl87Ndbw1WC{G9Pk!%eB5mu5@^n|K42dxfOKq!6x{tSdc=OY8-9UQtI$WKg zfQWLnp{Elljj)rbZxeTG#eE!qpx|V%_dnj4#*A3z_18HcrAl0|moQ)PJ_rIzGqnD+ zSfJc(Zh|U9ar)r!?BC>EGHz9+c7uV=$7>hFlQUrx_qz z{O10l=}=pg@vlPSUnhPDITp9cQ}UD4>HpiJj{!{l$7n}RC~?VO2Ee~P`t52c^Z5V! zp@{!aI|2w!Leu_dnCysAh=P(s6k3C~oEmuk*%CnywDj!9A^rdN1OloIY`}_gfAYs0US(bs>Ut}752F>^1v^6-DQ{Btb-=ji}zJ1|HmPmHMlcNdsp^$-Cq|8IaGqqEQ8 z|9gDQSVvx_Fmu|JmnKqonWubl`fM1QS3o0%Y1(f&yd~*J%Ga z%}7xp5Nu{QMDU-}>;-B9QXKKj{B!g}rGU|wYq62c{m;LL`f`9jqc)TI`Ca2bzZx|x z;HzO<5xQdjkA?d0dGKhp&eE~8pFejESim;QJ5 zk^2K35&C}EPwC$?DM0bpxFX~5kvOA|yb&hqf2A@2+4GcbyK(A&J-xpP_z!T|xNVAK zwer~t^|uDn#p`up+l^E=bqe3Tx*3}vjy2CKO869SQM?N1w1MrgYZ?KHtWg61DmL=Y zeOyaPNS2&>yh|4g8S|<9=k}gu6g-J{`JcW9hoWKLn1|5E6DOT#qM(VL;!TMykG!H{gc z!dBZ_ZvkU462o|JTuv%YeL^T&LRv7H_?ID_{~R6@n}YWfrmo0#mT>Mb&(5S{&>@_HFhWFsB z#V-Kj;M*V%Kn&RhToKXN5=Zc8Dvr$%^>eLaqpSt zMam(ny9!{dYUmwBkse5Ve*&b3nhbz*(t-EHc{{Kgz)~DI0y)|~B7on?agZ56#Hnz2 zi3iT?PX2Mwe9%VfhoQHCr$&rEFSfTRW<1$7Tou_KK{CdATK)lmWDxK+{Q4v?&>I>F zi1c5G1#cmj)O^P%-(KeZJt{=*0929ps1q)V`Ef?UR!Ew)g@v;R80`AmE{9!gP!ClT zr)&7=2k@TYX&pV|xYl3Tys+yAG%%`^(Ax@d2%dpUs|Pseoaz^-nr;44Lr~%RK6tTV zkf#|TGPG!&U>x6ioOTd#cWv~h1Di(79Y`$jlE*oQK8){!0x}#kLVyd=uOOZg!G=Kw zZq+>SjRfRS&Vr2Xbx$xR97)~siXSaBJZKpH!!-Vw1fO2gbZ)~gf zs5yyEnrhGKeu5<-ABID3^muwmLs)q-$@^{X4ACGFPRyVM)(Cx8nVD^raUVv>LKPr^ zA*vTGex!8Pj!7&hC`UAYtRk3#AN_69Eo&nvPG;c4j~|Qi+1*tfhG&53DX)VfUkTy> zvFv5=b~O5ge~seFJO=nEkHVsJBHM+4$bIt03k!fu+%OnlJt0k8SgbfiR6K77V0P}9 z=+Ct(8i8x*4)zpGM!N6_$`DA3(805c^~bPXwi55#~M+qNwTXd3u_30>vn=*;m@UPoJ0$zY{4lHZ5F39r!CC|^G1VdbU`KgMbl+Ft6Jdh@yZ zOv5)P$)y*}nN!PyE{$tc75n$6Ub$L}N`Hi^dGE9C73PoUnx^pF#RV>I5~o1~LQRgo zBv_%K3e_l!mK=%sSEf=1hPQpp=2?HjO|LC*1Q0}2%|@u2VQ|OMC|xWlT$Q5uCI zIszDJOQU@#2y z9rUoIVs~WRWToR;02`gg_%gx$!jF|JKBbR8Ph{)S=EZ0Ob0+jQu_uCYa$oVoZ%}>x z>IThgEGvx5X+Vs9tTF#kpWYf8Lkg0|`tCP(ctO7BNwKqJpC%OAW||ERe@I`Te!Qh9 zdQ!~OdNMJP4f%wggMIiIIbGjIsf$U9tA-^Iq{M9Dx69wby4()J#DigX1_EM9@1@(b z`YvLSef_*cNu@d-c`yS_pu7wSQpzj?>^ zqEIw)Q(BTZQDN-47FQC!WmufC*2E1SbHVyPa;NTHU?gCCqp-C0TpT00A=Lxe==SqM zqtZsyd|rc|qkZ=~$fA$7*%Pa%gBm)=>`y2`j3<3npztt4>Mir1s8Y@@Kmd_G>!faj zByxfta}!vU{lR3sklIyx+_ag^{0U}oDM!FvDW#wkh~3GO1)knbDS4ma21VfIC~YHOgfy=vjO<}5q$Y+-pqjdFl01L*$lr*VbG1Z$K>1K=M`X+o!8f;5_{ z&>YC~yJEdohxeS4zmkWX*@X!kXk}kr7Zs!31bn2rh&`8);m#I-D#97gKF zym6YagtGcYuW+Kw3xA4GcqAat&#K)8SXG`h9fIr_4(#U>ChIM$T=@Q|eLo|fIZn?R zsO9856ia4k2g*SSIKEW0*1`55Dq{c9}q!p4Q$d{6E4Qyvu8Q5XVh10bT(%;nlD)QqhyZS(}h2RrJ zrG>sA*p&Li%l&b6JKh|vCF$?SxiRWZ3I}%>kwMeGbO?esNzddzLd(v4v#2;e#NXqe zjQjj&9PfnGAGo{o@`@PP%|`P!Z{IM6?-^*Stw44nFm6zy_&p#?ItTsXlwj<4ITY~& zg?lm=^=hJ2oMY6U1};v!y|P77Vd&5a>;WydEnrG)cm`UjQK&Smwd(i#oxutbNO)KKm0VS?x?T4g$G>xB%)KwH1nQKmfJ6N&)PJF*Q zaFT)v&~{UWerhfX@|mL?16#OFZ9S`Q0{gC(HPwX^EfWHl28NvvI90)dPd~ z+&J^lZ-ujWw)0CVJG-tmH1O#=+xiW#ZT*1)05`hN;bVb?3~zr_koq=HMcg3m*#9>E z=fcDV6L4F|R+Z?taRs`d%gt`8SKQtJ+KD#olw(d-63C@h3j=S>_mmIxFaS#nG8`!X z6}a6(-iIw&?N7fpi)a;Rg%k3kf><9qCXH*2LPqf;;!*Ig9;-O~9enmwk{QEi^!WvR z#*sooCz`Vc?aYn>Rc~P)Pa-f|0IGv?;u3K4#*P4#pf)V6b>E``NG+naB(R9S4~i(3 zCIP+r^BhsP!=kV*l!LTOozAU%JP_hu$4^+%} zJSus282e*`Uwd*KulYyrcd`?Xk-0tG9HzSrK9F!&}_9 zYD3BIW>P>unQlKb{BGD?2)uHKT9zIFfxh=vz$LYB;@v{8MYAEr?K>2Oj_{kLd}r~T z_tMqkpS5Gat<21(kFrY{Gm65f`(%1xUvQM@s* zG5lcuroI$O@6mxxee#mQ0jSdx$Lv7v5^ZhKF)aK)-9iEKisBLy!7c%RTrpi_x9C!p)vvJV1~ObE{HRU?|sE?h|#7~=g(5$@L}Hg z=j#vW@YfL6?`dy&fm4Rk%r$0Lkf#@Dh{gOd>EsE->xGW*#rli0dt9X?>M@=YRF+I5 z<`U%oIs+AqH`)F)N~p8YcSD;dW$ z3Gn#t8=R2_&8cX+^Aj+YJjhc2saa`>!;cd%(rx>$`_@EQou8+sAyfs8ARo|-P*(%b zS8`(Q`I9@yIx4)@v~JbYZq)7PckEZ{T(aFW5mJn#WA~<0YUkZgQBcIecQbvDwvPQn zwtdPfS6WxXxO&h4e67Cz{cnI?>XHs~lBao>Yy%=wde4F{a>_-QDCREH_SufX{f*+% zFEwO+4q^!SLT%c+MUh**n_Ox56(kt)#yY>;;L;m7@-ViAeI8ZV<=S)pyf4d$eDU?y zCxYM(61FxHA{$^!-7#yr7!6Dk)iL-09;u8;95h0TyM;^p`K&+NrQDaj5zO>^xpz{A zqM3A7y{QO8Fj8^-l+XS~s&7!h=+tu9Gk(6w{taL{)&NJ?M&>WxfV~gP64O$Lb>3s! zMXh=)4WLIZ8AR}}J#c*pOEagE9Ba0)rCAT_r){$0mK)%u@k-bt9#xo}YfxES_}ROzgNxJ{f5@Wc(0++Zd}tp$3LDoa zTyn{ggQ(d1D4q5k3R_Z#e^To0FG#2CuZXyPJL^mS~ptCS!C0Ip9_ zPM!%z3umdexm$fvApJ0+K{!mZMYvA&B;hPGEow#JBzwe=6bOOe9iYz9UPB$Yj7Eze zZS)~xYHBNgDWYn4Vuc|O-YP+!*Lo683m))m3yNtCizyeW>^@*;`M~ya7{TL0a|7Tw zBL~%{%>X-rh{qa(*mP*v#ulw2n+Jpg_UOsEBy=eGiq1alG%KUy+No(P|EB}&5C4`u zTN`Dl*Mj5}!@joXL7s0ezpo=By02v$$NFy?i_LS&fNDEr&)1BuUlwgAdA#@Yi%W4% z_*kDjT;e zY=`>u?TbhAhj;sG&jcogD2eX`FfgU1N?2^|`A=g3am_7W+0$r>RiV4WP9 z2Pq**JlaDWa!kBz!chI^vMrxF@MNyDmDE1g&+K96$95pAk>Nb5x#S)IKn_*KUj2@NVrZvoigdmydM3dX{Px5AM&b8n)6>|* zr7hrCw~=ppy+U0-#WQHJz>}~5W|KN0JxL(z_yxw1`=RLb`L>(WL*QX>;ic$Z`1;(I z@Giiy7`jjXt=dnCF689r_<=}*((_;j_F$>7n+FSD_`OEl(#bu+^Mh8DT6E#YH7J{i zyQFqgHSz`S8)tOOpCP{97UPK4CoNQIer&Sw7>Byh4E@rsF&v_v9VBURq1|XGNn^n`0UO=AJ&OAG zj<%_#SLzhU*=-EM;jrt2)NTe4&v+h)bqW5LlQ=6DHS(R0ITGmH|qrTQ)EFswhEJA!r6F$Sg)%Dx#VvX8DE z)GYb%ZJR6D_8m>$McLxvYCMO{3rIKxEQf#YlV^u7dXIjx-qpB}6?+{0OGGSC|D=?~ z3WdJv;Skaz=m}0^8HhO^QTQfJanDsBNq+AxbRrwiiS`*MTlB7l8$Fy2@xXj>Ge(WX zJYWA!W<~<3m>a|-ITK-Vzj@%q5;~g5(RNl>5ox7^`WAjxHIL-NL%Emewq-)+?dA-2&@|l+RSoON3v2qIVlf z+H7$G2I}cNe;#ML%+&1_-!2WhR$t!NSZPrHyQSLK4<`!VovZ5!umIdR92gq8(cb%o zvW_dY=i&^LxW}QGrRqQAErZi^oOBoX9^bDbi>xBV>j@8~vdJF5`2?5Ne5yI+Kxfao z7@s#aB6ta=pZ?(o&|7T*%77v^&*Q<_XP6W-$RhipU_C{ZIN5r z2ivV?`G&Yo@!$<`C&|3EXP#~quXe3%(Q}C+`ICts zwR)gfL@tvd!uiokS*TV@4+h8d5bFRHGH{n?(qs-@wjlJ0s+ePbZ2I4H+iqWVCE9Tf z5ar6bC(`dZ+mzN?j$wpb=LYo4^ag$<>Q(X8wtVrF4ilq;z*BkG|gIL<7Xr zdiTl>cT2=4RRm)fwCQ{Ee6{~B25;{J0yZReY~-DQsbHCWzjw@bw%?jLD9yZw1bNSr zRo5B#Tv&n*fbc8Zw?-mvSZmz?xLCJc}Avy7G)1N5cSxqHKu6G?x&RaXW>JO#oF+JytDVW7PbFy&=L9xHz!X z7+R1EP{L-SXV*TBXS+}o6+7vj-kC(lbT_$5?60ydJ&%k#4^O%c2XdwMuGQb|6a;v# zh5u`>WtNr_1!rjLEOJg(xv9EX-ZB@(m+;x(DM??MKVaOcQlEBSL!<;9oSYVa^OFu{ zHYr({k)L(ti)62M3PPC*8|%HxlWJVTwYg)dq`+BDJZ zbdd$08}_!hUolSXuK_A#v5cP1%Wb2$h#u|6_^v24Ep$C2*KY8{2VXfYG-?OiD@f6^ z>!uj5l&x(np#nY(Oerar5Z<`Fc3$rCTa)wNFWPw{{eo7{7PLBB_td2E=QTy9nbCjyG z&m|Wvmn7I`$5sHnQ*B0V%*_xCLBkUKRa)x*LGeCBY`dW)cAs%P&{=cKW+>7b`)DcA zRgvnaU=kivS9Cer^!k`LT&TZS(6!|pNNp!+h(7)>@$qeZl&w8lqdh+pT==uyAZ7ClFmXh20dTsoE=`wLaIa+Y(3|N*L*BtyU0DKu)Gv z)T&Q4i^{JAz7sAuzUyFO>f#A6G2JUXuWnsju%X4g!$wTA3{ap%S zmOhkshKYV6W?sH|yjhaX#qRrE(HRJLu7qrOvY@Bnx{`{by9^J2e|?)n=fOWYb)Od7U;ygAoD!xll(!3S+Ee{J%VZ*3~Hhz@z6xOW<5u7gSj)NL%k#j7f7SFqn7wj!W z(Z}5)ZV%k30RB&!Z>Qb)O~S*VR+A)`_WC6{b2 z(@31;IxD8gB}8RsmU5Xk*P3YjzB}jqaUS*k_x*T(-;dAb{rEoK-_P^&dh@VJqc*kc zo$le`TbS3chS(l{l%7I=2I!z$I1Yep&S*y z;rZZh*e!-AxR~8b!%pK;mIJ5F#E~ju^G4?=%ef0U-E;}+#_`&UE(M!n@ZdEc22od| zS1z91q})SX?~NKASaZ}j`F+z{MGd99)aA}NoPFMv3pkNhkIjxYQccnbI>-Q33I@vR zx3HTZ$0TIvRAl?V!KjIuqWoWnBZdPmFG~qbqL|6scP{*EZH1cJ#oNIOZR^1>{>y$){&M1&5iss z6#M!7Q*BH|ym^DD3GE=4wB^Zir$$4ftyc@FMtzHQ-=OS>wIGnC#J?W;sUKUpWDSN$z_2e)%Y23H==l{;K=l_hdXN51B;W*Jx zENSNeiII0kowY+U(Bj8`kio;&Q;X(7?joNOrh&Yk{>qg{rQVq5p zjWa;C$d+*Aw%t>gye@EZgD&Guo-7MH%{CL%#O4WPd8sj5Qoi=Gr`6|en8#D-P7;^I;CTz8?#Hv;mR3-9qlXp-qA%+p24x3?6obmN zz!|3LzQ~r{32xnDsG_hnbo75fkkPg}Z$qYW za1q;FpB2u8&RTek@Q534_V*hooUuTmHlMvcZqk!kz=KCssLr@JzPsdb_w<`NbY=~= zcwz|)4?SGFPp}<0biTg!#NaDdB+^RPV6VVXHhVG01-8Wcm<7it8#<2E&hO^Ny9NztvDZ>#W4U(jQVR* zPvVN)cOMo?TALj6Ofr@+>(aRVcl{!GY@lKdO@?$Zt3e%csNB592d#*|Kv#FhA5R z`0}K+oS%VaY+OrR^d2zl6qDxyBDTiFL$zMKglBpU?T5r8_BcVDmEY@iKiUy{T!S|D z5nviyd-$nRqhVPeJewVHz0Xqs6=p#3CiDu%FP2KoV$osKX4C$n#ZX3KkwIrnzsgKUMY||TrjExq!8*snAj`JwV=#~RdVrWbJM;; z+*G0>G>Zk+u|i2C={d;|v9cJ!5a9zuR|ZuT+C;i7gvg7uGsV-A_q+x3Zc2^3cgR8T z6Gl^qqPC$tQ^+|OLZZizD`mf(zyUx(I!uNuz|Bvn=iCEOmMQ4x7W4<;!J-BJFG zdz#}eI0m48wJ_QC{#z-Sf|g+d>)cR=)?A@=@~*5iSdUrX^`q8%W5p= zG`IT{xe)!a6f!$&?fJYVh~$+*LAgzaT48Px`6YB;;MK-pZL!ssFi_vriS7RkKbz-0 z`9~c?@wUMxN`Z^qnvrbkah8u|Gj?3x3-6N1P859xERZ`&DwCnrwVW0JrStF5I7b@~ zL7YGAqUxJey3TK=rouICaxj1k@z!4qwp#BEnC*iem6p5Z^vW_07M|{T)ERQ1b8vK3 zFW?+LQ-gcl(zpLQC^EJW<&cu;Zb%)~Ct!}x#aWu_1bnn}D@wgNhr?*6x?y#kTM4%M zCloaHOVc=HU?IU3_8OaygK6u(csts7McxLB_#Cv)4Fi@_Jjsl)_f-cwR)Espb!br_!q3iZqOnVPGP>9XN^*6@ThH9DiVie$B*Q3E49qc29Q*fp&zT zfAiB<^zQN-vMrLMSLco`N0|;ZyCK6o@HMMQ#SM+`g=8e|cEX~&r7wh$q%xice4;&I z$Z5$ND2;wOPdyD6PH5FVX@+hkqZ|#5zF`oz?)UAPIOL0X$p^0|FIIAFfWht+cWG(g zFM!iYr;JOW#C(wOYYes7D7-N{r$V<_Iho}78UR`^Fq;pT>=E6(vtmwA-ZpmSi$X+V zL3~S6RNm?71&D9#{3rYR+Cu_Wz}PMqQ&45iaLke#wlb$&#a5@=!TYrn`q0umhGxB) zbCLDb5*gX;A}cD`N4H$Fy_9R)dF8+&E#ukwsg2B$ZsR8w&8%Or=Nh9*Z(K^9tdH(u z!Md*#ir4q;p`F@FTzHEhN`??e^K;uDjanb$*1*>r@0S;&5+faXc+F`2KEw;(b7&Pb z&8qLqpx;A-?BETRg}gq!yw?T_OmI26R^{|B6K9&=E|#yQ?2mHc=^2zQk#(Bxp9|4^ z+fve?I@qVYRuQ^lw6--UYv@3&v}RS9b_D#jb>8i9f*`h@>;hH^OCC=tH3^ffsxR_u zZ~xeAZo?)SZTynCknnwpgrdoi&bG(z(LZp=yJeO;*h!@ynBomGl~`I>Yvd0ca None: self.graph = graph self.topics = topics self.stack = AsyncExitStack() self.kwargs = kwargs + self.consumer_kwargs = consumer_kwargs or {} + self.producer_kwargs = producer_kwargs or {} self.group_id = group_id self.batch_max_n = batch_max_n self.batch_max_ms = batch_max_ms diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index 9d548280d..a3da188e8 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -38,12 +38,16 @@ class KafkaOrchestrator(AbstractAsyncContextManager): batch_max_n: int = 10, batch_max_ms: int = 1000, retry_policy: Optional[RetryPolicy] = None, + consumer_kwargs: Optional[dict[str, Any]] = None, + producer_kwargs: Optional[dict[str, Any]] = None, **kwargs: Any, ) -> None: self.graph = graph self.topics = topics self.stack = AsyncExitStack() self.kwargs = kwargs + self.consumer_kwargs = consumer_kwargs or {} + self.producer_kwargs = producer_kwargs or {} self.group_id = group_id self.batch_max_n = batch_max_n self.batch_max_ms = batch_max_ms @@ -57,10 +61,15 @@ class KafkaOrchestrator(AbstractAsyncContextManager): group_id=self.group_id, enable_auto_commit=False, **self.kwargs, + **self.consumer_kwargs, ) ) self.producer = await self.stack.enter_async_context( - aiokafka.AIOKafkaProducer(value_serializer=serde.dumps, **self.kwargs) + aiokafka.AIOKafkaProducer( + value_serializer=serde.dumps, + **self.kwargs, + **self.producer_kwargs, + ) ) self.subgraphs = { k: v async for k, v in self.graph.aget_subgraphs(recurse=True) diff --git a/libs/scheduler-kafka/tests/drain.py b/libs/scheduler-kafka/tests/drain.py index b58906362..b6199adbb 100644 --- a/libs/scheduler-kafka/tests/drain.py +++ b/libs/scheduler-kafka/tests/drain.py @@ -1,10 +1,11 @@ import asyncio import functools -from typing import Callable, Optional, ParamSpec, TypeVar +from typing import Callable, Optional, TypeVar import anyio from aiokafka import AIOKafkaConsumer from langchain_core.runnables import RunnableConfig +from typing_extensions import ParamSpec from langgraph.pregel import Pregel from langgraph.pregel.types import StateSnapshot diff --git a/libs/scheduler-kafka/tests/test_subgraph.py b/libs/scheduler-kafka/tests/test_subgraph.py index a9c0062a8..8c8a0f530 100644 --- a/libs/scheduler-kafka/tests/test_subgraph.py +++ b/libs/scheduler-kafka/tests/test_subgraph.py @@ -1,4 +1,4 @@ -from typing import Literal, ParamSpec, TypeVar, cast +from typing import Literal, cast import pytest from aiokafka import AIOKafkaProducer @@ -19,8 +19,6 @@ from tests.any import AnyDict, AnyStr from tests.drain import drain_topics pytestmark = pytest.mark.anyio -C = ParamSpec("C") -R = TypeVar("R") def mk_weather_graph(checkpointer: BaseCheckpointSaver) -> Pregel: From 5e0273886e10180e4557d6e371d4ba39a98d984b Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 16:02:47 -0700 Subject: [PATCH 25/34] Lint --- libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py | 3 ++- libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py index 8aec3ef9e..86812c258 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/executor.py @@ -1,11 +1,12 @@ import asyncio from contextlib import AbstractAsyncContextManager, AsyncExitStack from functools import partial -from typing import Any, Optional, Self, Sequence +from typing import Any, Optional, Sequence import aiokafka import orjson from langchain_core.runnables import RunnableConfig +from typing_extensions import Self import langgraph.scheduler.kafka.serde as serde from langgraph.constants import CONFIG_KEY_DELEGATE, ERROR, NS_END, NS_SEP diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py index a3da188e8..c13de9c08 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/orchestrator.py @@ -1,9 +1,10 @@ import asyncio from contextlib import AbstractAsyncContextManager, AsyncExitStack -from typing import Any, Optional, Self +from typing import Any, Optional import aiokafka from langchain_core.runnables import ensure_config +from typing_extensions import Self import langgraph.scheduler.kafka.serde as serde from langgraph.constants import ( From 3a6546076791ebabfa03479430172e3e3d0e8e54 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 16:10:03 -0700 Subject: [PATCH 26/34] Use anyio for timeout in dran test helper --- libs/scheduler-kafka/tests/drain.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/libs/scheduler-kafka/tests/drain.py b/libs/scheduler-kafka/tests/drain.py index b6199adbb..2a6ed426b 100644 --- a/libs/scheduler-kafka/tests/drain.py +++ b/libs/scheduler-kafka/tests/drain.py @@ -1,5 +1,4 @@ import asyncio -import functools from typing import Callable, Optional, TypeVar import anyio @@ -17,19 +16,6 @@ C = ParamSpec("C") R = TypeVar("R") -def timeout(delay: int): - def decorator(func: Callable[C, R]) -> Callable[C, R]: - @functools.wraps(func) - async def new_func(*args: C.args, **kwargs: C.kwargs) -> R: - async with asyncio.timeout(delay): - return await func(*args, **kwargs) - - return new_func - - return decorator - - -@timeout(20) async def drain_topics( topics: Topics, graph: Pregel, @@ -79,6 +65,7 @@ async def drain_topics( # run the orchestrator and executor until break_when async with anyio.create_task_group() as tg: + tg.cancel_scope.deadline = anyio.current_time() + 20 scope = tg.cancel_scope tg.start_soon(orchestrator, name="orchestrator") tg.start_soon(executor, name="executor") From 3886b76771d08e67c4ebf1ec361f105692f686f3 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 16:12:17 -0700 Subject: [PATCH 27/34] Lint --- libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py b/libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py index 52ad044b7..6f3137e68 100644 --- a/libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py +++ b/libs/scheduler-kafka/langgraph/scheduler/kafka/retry.py @@ -1,7 +1,9 @@ import asyncio import logging import random -from typing import Awaitable, Callable, Optional, ParamSpec +from typing import Awaitable, Callable, Optional + +from typing_extensions import ParamSpec from langgraph.pregel.types import RetryPolicy From edc0fd021037e8ee073749b1084d543725634103 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 16:20:45 -0700 Subject: [PATCH 28/34] Remove extraneous asserts --- libs/scheduler-kafka/tests/test_subgraph.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/scheduler-kafka/tests/test_subgraph.py b/libs/scheduler-kafka/tests/test_subgraph.py index 8c8a0f530..fa4ec50d3 100644 --- a/libs/scheduler-kafka/tests/test_subgraph.py +++ b/libs/scheduler-kafka/tests/test_subgraph.py @@ -138,8 +138,6 @@ async def test_subgraph_w_interrupt( # check interrupted state state = await graph.aget_state(config) - assert len(orch_msgs) == 6 - assert len(exec_msgs) == 5 assert state.next == ("weather_graph",) assert state.values == { "messages": [HumanMessage(id=AnyStr(), content="what's the weather in sf")], @@ -427,8 +425,6 @@ async def test_subgraph_w_interrupt( # check final state state = await graph.aget_state(config) - assert len(orch_msgs) == 4 - assert len(exec_msgs) == 3 assert state.next == () assert state.values == { "messages": [ From bdabb723b06b28d703e6a5389d1aca257c447db0 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 16:39:34 -0700 Subject: [PATCH 29/34] Add langgraph dep --- libs/scheduler-kafka/poetry.lock | 4 ++-- libs/scheduler-kafka/pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/scheduler-kafka/poetry.lock b/libs/scheduler-kafka/poetry.lock index 0c84bbc33..0d1a89d12 100644 --- a/libs/scheduler-kafka/poetry.lock +++ b/libs/scheduler-kafka/poetry.lock @@ -511,7 +511,7 @@ typing-extensions = ">=4.7" [[package]] name = "langgraph" -version = "0.2.17" +version = "0.2.19" description = "Building stateful, multi-actor applications with LLMs" optional = false python-versions = ">=3.9.0,<4.0" @@ -1179,4 +1179,4 @@ watchmedo = ["PyYAML (>=3.10)"] [metadata] lock-version = "2.0" python-versions = "^3.9.0,<4.0" -content-hash = "29e1bd946c9d7c9424219fa94fca03c80514d98c58e9fc5a7d52f735b8d0d8c9" +content-hash = "4fd0a2d16956a5e92ef42cbd23a1649cc5cefcc2da1d58d0065fa355668dfaa9" diff --git a/libs/scheduler-kafka/pyproject.toml b/libs/scheduler-kafka/pyproject.toml index 49bc0b499..84f7b7fb3 100644 --- a/libs/scheduler-kafka/pyproject.toml +++ b/libs/scheduler-kafka/pyproject.toml @@ -13,6 +13,7 @@ python = "^3.9.0,<4.0" orjson = "^3.10.7" crc32c = "^2.7.post1" aiokafka = "^0.11.0" +langgraph = "^0.2.19" [tool.poetry.group.dev.dependencies] ruff = "^0.6.2" From bb10acbe2862ad0f62f6f7023d4ac8808a1a8d28 Mon Sep 17 00:00:00 2001 From: vbarda Date: Tue, 10 Sep 2024 19:51:23 -0400 Subject: [PATCH 30/34] update cache keys --- .github/workflows/_lint.yml | 2 +- .github/workflows/_test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.yml index 9110da7c8..9ba0ff5a9 100644 --- a/.github/workflows/_lint.yml +++ b/.github/workflows/_lint.yml @@ -44,7 +44,7 @@ jobs: python-version: ${{ matrix.python-version }} poetry-version: ${{ env.POETRY_VERSION }} working-directory: ${{ inputs.working-directory }} - cache-key: lint-with-extras + cache-key: lint-${{ inputs.working-directory }} - name: Check Poetry File if: steps.changed-files.outputs.all diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 913941463..ea8849b95 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -34,7 +34,7 @@ jobs: python-version: ${{ matrix.python-version }} poetry-version: ${{ env.POETRY_VERSION }} working-directory: ${{ inputs.working-directory }} - cache-key: core + cache-key: test-${{ inputs.working-directory }} - name: Install dependencies shell: bash From 5de46f7472de8e7b4854b7f0add5d2d13f681b17 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 16:51:49 -0700 Subject: [PATCH 31/34] Update stop condition for drain --- libs/scheduler-kafka/tests/drain.py | 46 +++++++++------------ libs/scheduler-kafka/tests/test_fanout.py | 12 ++---- libs/scheduler-kafka/tests/test_subgraph.py | 11 +---- 3 files changed, 25 insertions(+), 44 deletions(-) diff --git a/libs/scheduler-kafka/tests/drain.py b/libs/scheduler-kafka/tests/drain.py index 2a6ed426b..21fcd4d4c 100644 --- a/libs/scheduler-kafka/tests/drain.py +++ b/libs/scheduler-kafka/tests/drain.py @@ -1,13 +1,11 @@ import asyncio -from typing import Callable, Optional, TypeVar +from typing import Optional, TypeVar import anyio from aiokafka import AIOKafkaConsumer -from langchain_core.runnables import RunnableConfig from typing_extensions import ParamSpec from langgraph.pregel import Pregel -from langgraph.pregel.types import StateSnapshot from langgraph.scheduler.kafka.executor import KafkaExecutor from langgraph.scheduler.kafka.orchestrator import KafkaOrchestrator from langgraph.scheduler.kafka.types import MessageToOrchestrator, Topics @@ -17,31 +15,38 @@ R = TypeVar("R") async def drain_topics( - topics: Topics, - graph: Pregel, - config: RunnableConfig, - *, - until: Callable[[StateSnapshot], bool], - debug: bool = False, + topics: Topics, graph: Pregel, *, debug: bool = False ) -> tuple[list[MessageToOrchestrator], list[MessageToOrchestrator]]: scope: Optional[anyio.CancelScope] = None orch_msgs = [] exec_msgs = [] errors = [] + def done() -> bool: + return ( + len(orch_msgs) > 0 + and len(exec_msgs) > 0 + and not orch_msgs[-1] + and not exec_msgs[-1] + ) + async def orchestrator() -> None: async with KafkaOrchestrator(graph, topics) as orch: async for msgs in orch: - orch_msgs.extend(msgs) + orch_msgs.append(msgs) if debug: print("\n---\norch", len(msgs), msgs) + if done(): + scope.cancel() async def executor() -> None: async with KafkaExecutor(graph, topics) as exec: async for msgs in exec: - exec_msgs.extend(msgs) + exec_msgs.append(msgs) if debug: print("\n---\nexec", len(msgs), msgs) + if done(): + scope.cancel() async def error_consumer() -> None: async with AIOKafkaConsumer(topics.error) as consumer: @@ -50,18 +55,8 @@ async def drain_topics( if scope: scope.cancel() - async def poller(expected_next: tuple[str, ...]) -> None: - while True: - await asyncio.sleep(0.5) - state = await graph.aget_state(config) - if until(state): - break - if scope: - scope.cancel() - - # start error consumer and poller + # start error consumer error_task = asyncio.create_task(error_consumer(), name="error_consumer") - poller_task = asyncio.create_task(poller(()), name="poller") # run the orchestrator and executor until break_when async with anyio.create_task_group() as tg: @@ -70,16 +65,15 @@ async def drain_topics( tg.start_soon(orchestrator, name="orchestrator") tg.start_soon(executor, name="executor") - # cancel error consumer and poller + # cancel error consumer error_task.cancel() - poller_task.cancel() try: - await asyncio.gather(error_task, poller_task) + await error_task except asyncio.CancelledError: pass # check no errors assert not errors, errors - return orch_msgs, exec_msgs + return [m for mm in orch_msgs for m in mm], [m for mm in exec_msgs for m in mm] diff --git a/libs/scheduler-kafka/tests/test_fanout.py b/libs/scheduler-kafka/tests/test_fanout.py index 0ecc3d80d..9f3726648 100644 --- a/libs/scheduler-kafka/tests/test_fanout.py +++ b/libs/scheduler-kafka/tests/test_fanout.py @@ -99,9 +99,7 @@ async def test_fanout_graph(topics: Topics, checkpointer: BaseCheckpointSaver) - ) # drain topics - orch_msgs, exec_msgs = await drain_topics( - topics, graph, config, until=lambda s: s.values and s.next == () - ) + orch_msgs, exec_msgs = await drain_topics(topics, graph) # check state state = await graph.aget_state(config) @@ -184,9 +182,7 @@ async def test_fanout_graph_w_interrupt( MessageToOrchestrator(input=input, config=config), ) - orch_msgs, exec_msgs = await drain_topics( - topics, graph, config, until=lambda s: s.values and s.next == ("qa",) - ) + orch_msgs, exec_msgs = await drain_topics(topics, graph) # check interrupted state state = await graph.aget_state(config) @@ -262,9 +258,7 @@ async def test_fanout_graph_w_interrupt( MessageToOrchestrator(input=None, config=config), ) - orch_msgs, exec_msgs = await drain_topics( - topics, graph, config, until=lambda s: s.values and s.next == () - ) + orch_msgs, exec_msgs = await drain_topics(topics, graph) # check final state state = await graph.aget_state(config) diff --git a/libs/scheduler-kafka/tests/test_subgraph.py b/libs/scheduler-kafka/tests/test_subgraph.py index fa4ec50d3..ab3b4d20c 100644 --- a/libs/scheduler-kafka/tests/test_subgraph.py +++ b/libs/scheduler-kafka/tests/test_subgraph.py @@ -129,12 +129,7 @@ async def test_subgraph_w_interrupt( MessageToOrchestrator(input=input, config=config), ) - orch_msgs, exec_msgs = await drain_topics( - topics, - graph, - config, - until=lambda state: state.next == ("weather_graph",), - ) + orch_msgs, exec_msgs = await drain_topics(topics, graph) # check interrupted state state = await graph.aget_state(config) @@ -419,9 +414,7 @@ async def test_subgraph_w_interrupt( MessageToOrchestrator(input=None, config=config), ) - orch_msgs, exec_msgs = await drain_topics( - topics, graph, config, until=lambda state: state.next == () - ) + orch_msgs, exec_msgs = await drain_topics(topics, graph) # check final state state = await graph.aget_state(config) From 77083bf634a3facca63137f892a71f833be7484d Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 17:22:06 -0700 Subject: [PATCH 32/34] Disable test jobs for now --- .github/workflows/ci.yml | 218 +++++++++++++++++++-------------------- 1 file changed, 107 insertions(+), 111 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2251d9f79..e16649d47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,116 +1,112 @@ --- - name: CI +name: CI - on: - push: - branches: [main] - pull_request: +on: + push: + branches: [main] + pull_request: - # If another push to the same PR or branch happens while this workflow is still running, - # cancel the earlier run in favor of the next run. - # - # There's no point in testing an outdated version of the code. GitHub only allows - # a limited number of job runners to be active at the same time, so it's better to cancel - # pointless jobs early so that more useful jobs can run sooner. - concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# If another push to the same PR or branch happens while this workflow is still running, +# cancel the earlier run in favor of the next run. +# +# There's no point in testing an outdated version of the code. GitHub only allows +# a limited number of job runners to be active at the same time, so it's better to cancel +# pointless jobs early so that more useful jobs can run sooner. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +env: + POETRY_VERSION: "1.7.1" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + lint: + name: cd ${{ matrix.working-directory }} + strategy: + matrix: + working-directory: + [ + "libs/langgraph", + "libs/sdk-py", + "libs/cli", + "libs/checkpoint", + "libs/checkpoint-sqlite", + "libs/checkpoint-postgres", + "libs/scheduler-kafka", + ] + uses: ./.github/workflows/_lint.yml + with: + working-directory: ${{ matrix.working-directory }} + secrets: inherit + + test: + name: cd ${{ matrix.working-directory }} + strategy: + matrix: + working-directory: [ + "libs/langgraph", + "libs/cli", + "libs/checkpoint", + "libs/checkpoint-sqlite", + "libs/checkpoint-postgres", + # "libs/scheduler-kafka" + ] + uses: ./.github/workflows/_test.yml + with: + working-directory: ${{ matrix.working-directory }} + secrets: inherit + + integration-test: + name: CLI integration test + uses: ./.github/workflows/_integration_test.yml + secrets: inherit + + lint-js: + runs-on: ubuntu-latest + strategy: + matrix: + working-directory: + - "libs/sdk-js" + defaults: + run: + working-directory: ${{ matrix.working-directory }} + steps: + - uses: actions/checkout@v3 + - name: Setup Node.js (LTS) + uses: actions/setup-node@v3 + with: + node-version: "20" + cache: "yarn" + cache-dependency-path: ${{ matrix.working-directory }}/yarn.lock + - name: Install dependencies + run: yarn install + - name: Run lint + run: yarn lint + - name: Build + run: yarn build + + ci_success: + name: "CI Success" + needs: [build, lint, lint-js, test, integration-test] + if: | + always() + runs-on: ubuntu-latest env: - POETRY_VERSION: "1.7.1" - - jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - lint: - name: cd ${{ matrix.working-directory }} - needs: [ build ] - strategy: - matrix: - working-directory: [ - "libs/langgraph", - "libs/sdk-py", - "libs/cli", - "libs/checkpoint", - "libs/checkpoint-sqlite", - "libs/checkpoint-postgres", - "libs/scheduler-kafka" - ] - uses: ./.github/workflows/_lint.yml - with: - working-directory: ${{ matrix.working-directory }} - secrets: inherit - - test: - name: cd ${{ matrix.working-directory }} - needs: [ build ] - strategy: - matrix: - working-directory: [ - "libs/langgraph", - "libs/cli", - "libs/checkpoint", - "libs/checkpoint-sqlite", - "libs/checkpoint-postgres", - "libs/scheduler-kafka" - ] - uses: ./.github/workflows/_test.yml - with: - working-directory: ${{ matrix.working-directory }} - secrets: inherit - - integration-test: - name: CLI integration test - needs: [ build ] - uses: ./.github/workflows/_integration_test.yml - secrets: inherit - - lint-js: - runs-on: ubuntu-latest - needs: [ build ] - strategy: - matrix: - working-directory: - - "libs/sdk-js" - defaults: - run: - working-directory: ${{ matrix.working-directory }} - steps: - - uses: actions/checkout@v3 - - name: Setup Node.js (LTS) - uses: actions/setup-node@v3 - with: - node-version: "20" - cache: "yarn" - cache-dependency-path: ${{ matrix.working-directory }}/yarn.lock - - name: Install dependencies - run: yarn install - - name: Run lint - run: yarn lint - - name: Build - run: yarn build - - ci_success: - name: "CI Success" - needs: [build, lint, lint-js, test, integration-test] - if: | - always() - runs-on: ubuntu-latest - env: - JOBS_JSON: ${{ toJSON(needs) }} - RESULTS_JSON: ${{ toJSON(needs.*.result) }} - EXIT_CODE: ${{!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && '0' || '1'}} - steps: - - name: "CI Success" - run: | - echo $JOBS_JSON - echo $RESULTS_JSON - echo "Exiting with $EXIT_CODE" - exit $EXIT_CODE - \ No newline at end of file + JOBS_JSON: ${{ toJSON(needs) }} + RESULTS_JSON: ${{ toJSON(needs.*.result) }} + EXIT_CODE: ${{!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && '0' || '1'}} + steps: + - name: "CI Success" + run: | + echo $JOBS_JSON + echo $RESULTS_JSON + echo "Exiting with $EXIT_CODE" + exit $EXIT_CODE From 2b09c284d41b3b02bdc61c6de01b4f26c33072dd Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 17:22:48 -0700 Subject: [PATCH 33/34] Remove empty build job --- .github/workflows/ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e16649d47..5cb4ce2bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,14 +20,6 @@ env: POETRY_VERSION: "1.7.1" jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.10" - lint: name: cd ${{ matrix.working-directory }} strategy: From 34d530d5d83837fa080c1db2d055fe952cfc8488 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 10 Sep 2024 17:24:24 -0700 Subject: [PATCH 34/34] Fix --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cb4ce2bf..48ee947c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,7 @@ jobs: ci_success: name: "CI Success" - needs: [build, lint, lint-js, test, integration-test] + needs: [lint, lint-js, test, integration-test] if: | always() runs-on: ubuntu-latest