From 1b9089cccaf9ad7a75a86d9911c91d2ff4e0d744 Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Tue, 12 Sep 2023 12:09:05 +0100 Subject: [PATCH] Add retry support --- README.md | 2 +- permchain/pubsub.py | 12 ++++++------ poetry.lock | 10 +++++----- pyproject.toml | 2 +- tests/test_invoke.py | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 6e42fa451..8d47fd85d 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Check `tests` and `examples` for more examples. ## Near-term Roadmap -- [ ] Add initial retry support (pending changes in `langchain`) +- [x] Add initial retry support (pending changes in `langchain`) - [ ] Detect cycles (aka. infinite loops) and throw an error - [ ] Allow user to catch that error (by subcribing to an error topic?) - [ ] Replace Queue data structure with a Log data structure (this will enable checking the status of the readers, etc.) diff --git a/permchain/pubsub.py b/permchain/pubsub.py index 14859fa3f..c307f0085 100644 --- a/permchain/pubsub.py +++ b/permchain/pubsub.py @@ -64,11 +64,11 @@ class PubSub(Serializable, Runnable[Any, Any], ABC): class Config: arbitrary_types_allowed = True - # def with_retry(self, retry: BaseRetrying) -> Runnable[Any, Any]: - # return self.__class__( - # processes=[p.with_retry(retry) for p in self.processes], - # connection=self.connection, - # ) + def with_retry(self, **kwargs: Any) -> Runnable[Any, Any]: + return self.__class__( + processes=[p.with_retry(**kwargs) for p in self.processes], + connection=self.connection, + ) def _transform( self, @@ -153,7 +153,7 @@ class PubSub(Serializable, Runnable[Any, Any], ABC): **patch_config( config, callbacks=run_manager.get_child(), - # run_name=f"Topic: {process.topic.name}", + run_name=f"Topic: {process.topic.name}", ), CONFIG_SEND_KEY: send, CONFIG_GET_KEY: get, diff --git a/poetry.lock b/poetry.lock index fea717b97..b66daf950 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1516,13 +1516,13 @@ files = [ [[package]] name = "langchain" -version = "0.0.275" +version = "0.0.286" description = "Building applications with LLMs through composability" optional = false python-versions = ">=3.8.1,<4.0" files = [ - {file = "langchain-0.0.275-py3-none-any.whl", hash = "sha256:ac1ff4dc95c0ebe8ca76535f444ce2e5324dec23f9732c880c7588d4d91f49ba"}, - {file = "langchain-0.0.275.tar.gz", hash = "sha256:23880ff4eb4b1d1f5defd1d904cd64a19d0f0a722307284906848ef1fec88eaf"}, + {file = "langchain-0.0.286-py3-none-any.whl", hash = "sha256:487caa0eb92abf8bab574429fbb064e76569510cc7225ab7a96ae8d40cc19585"}, + {file = "langchain-0.0.286.tar.gz", hash = "sha256:3a5dae7e8726ae755f3671105610911a7cf2d3e1fb4b0531b6e0b4336051ea2f"}, ] [package.dependencies] @@ -1545,7 +1545,7 @@ clarifai = ["clarifai (>=9.1.0)"] cohere = ["cohere (>=4,<5)"] docarray = ["docarray[hnswlib] (>=0.32.0,<0.33.0)"] embeddings = ["sentence-transformers (>=2,<3)"] -extended-testing = ["amazon-textract-caller (<2)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.0.7,<0.0.8)", "chardet (>=5.1.0,<6.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "openai (>=0,<1)", "openapi-schema-pydantic (>=1.2,<2.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "tqdm (>=4.48.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] +extended-testing = ["amazon-textract-caller (<2)", "assemblyai (>=0.17.0,<0.18.0)", "atlassian-python-api (>=3.36.0,<4.0.0)", "beautifulsoup4 (>=4,<5)", "bibtexparser (>=1.4.0,<2.0.0)", "cassio (>=0.1.0,<0.2.0)", "chardet (>=5.1.0,<6.0.0)", "dashvector (>=1.0.1,<2.0.0)", "esprima (>=4.0.1,<5.0.0)", "faiss-cpu (>=1,<2)", "feedparser (>=6.0.10,<7.0.0)", "geopandas (>=0.13.1,<0.14.0)", "gitpython (>=3.1.32,<4.0.0)", "gql (>=3.4.1,<4.0.0)", "html2text (>=2020.1.16,<2021.0.0)", "jinja2 (>=3,<4)", "jq (>=1.4.1,<2.0.0)", "lxml (>=4.9.2,<5.0.0)", "markdownify (>=0.11.6,<0.12.0)", "mwparserfromhell (>=0.6.4,<0.7.0)", "mwxml (>=0.3.3,<0.4.0)", "newspaper3k (>=0.2.8,<0.3.0)", "openai (>=0,<1)", "openapi-schema-pydantic (>=1.2,<2.0)", "pandas (>=2.0.1,<3.0.0)", "pdfminer-six (>=20221105,<20221106)", "pgvector (>=0.1.6,<0.2.0)", "psychicapi (>=0.8.0,<0.9.0)", "py-trello (>=0.19.0,<0.20.0)", "pymupdf (>=1.22.3,<2.0.0)", "pypdf (>=3.4.0,<4.0.0)", "pypdfium2 (>=4.10.0,<5.0.0)", "pyspark (>=3.4.0,<4.0.0)", "rank-bm25 (>=0.2.2,<0.3.0)", "rapidfuzz (>=3.1.1,<4.0.0)", "requests-toolbelt (>=1.0.0,<2.0.0)", "scikit-learn (>=1.2.2,<2.0.0)", "sqlite-vss (>=0.1.2,<0.2.0)", "streamlit (>=1.18.0,<2.0.0)", "sympy (>=1.12,<2.0)", "telethon (>=1.28.5,<2.0.0)", "tqdm (>=4.48.0)", "xata (>=1.0.0a7,<2.0.0)", "xmltodict (>=0.13.0,<0.14.0)"] javascript = ["esprima (>=4.0.1,<5.0.0)"] llms = ["clarifai (>=9.1.0)", "cohere (>=4,<5)", "huggingface_hub (>=0,<1)", "manifest-ml (>=0.0.1,<0.0.2)", "nlpcloud (>=1,<2)", "openai (>=0,<1)", "openlm (>=0.0.5,<0.0.6)", "torch (>=1,<3)", "transformers (>=4,<5)"] openai = ["openai (>=0,<1)", "tiktoken (>=0.3.2,<0.4.0)"] @@ -3517,4 +3517,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<4.0" -content-hash = "09dcfc3a9e1f44f36e412f2c4d0bd0e5822fdc8053e6a8e2c01b47ed96e7dc88" +content-hash = "0076068c47a6023f7be557c58b60a1e1f42401e1776fb6282dadc2be9ee77618" diff --git a/pyproject.toml b/pyproject.toml index d636388cf..131dce5ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ repository = "https://www.github.com/langchain-ai/permchain" [tool.poetry.dependencies] python = ">=3.8.1,<4.0" -langchain = "^0.0.275" +langchain = "^0.0.286" [tool.poetry.group.test.dependencies] diff --git a/tests/test_invoke.py b/tests/test_invoke.py index fe97e7a2e..f397f22ab 100644 --- a/tests/test_invoke.py +++ b/tests/test_invoke.py @@ -105,7 +105,7 @@ def test_invoke_two_processes_one_in_two_out(mocker: MockerFixture): topic_one = Topic("one") # Topic.publish() is passthrough so we can publish to multiple topics in sequence chain_one = ( - Topic.IN.subscribe() | add_one | topic_one.publish() | Topic.OUT.publish() + Topic.IN.subscribe() | add_one | Topic.OUT.publish() | topic_one.publish() ) chain_two = topic_one.subscribe() | add_one | Topic.OUT.publish()