From 67ee1d87c7680b94a8e9be50b2b2e52fd4f0542e Mon Sep 17 00:00:00 2001 From: Nuno Campos Date: Thu, 24 Aug 2023 17:21:53 +0200 Subject: [PATCH] Improve tracing output, update for newer runnables features, --- examples/example.ipynb | 34 ++++++++-- examples/run_over_dataset.py | 0 examples/web-research.ipynb | 119 ++++++++++++++++------------------- permchain/pubsub.py | 86 +++++++++++-------------- permchain/topic.py | 4 +- 5 files changed, 120 insertions(+), 123 deletions(-) create mode 100644 examples/run_over_dataset.py diff --git a/examples/example.ipynb b/examples/example.ipynb index cdf0ece54..9c514a1f3 100644 --- a/examples/example.ipynb +++ b/examples/example.ipynb @@ -69,7 +69,7 @@ { "data": { "text/plain": [ - "\"Arr, me matey, ye be askin' a question that be as vast as the seven seas. Art be a treasure, a form o' expression that be capturin' the heart and soul. It be the brushstrokes on a canvas, the melodies in a shanty, the words on a page. Art be a glimpse into the mind o' the artist, a window into their world. It be a means o' communicatin', stirrin' emotions, and provokin' thoughts. It be subjective, me hearties, for what be art to one may not be art to another. But in the end, art be a gift that be enrichin' our lives and feedin' our souls.\"" + "\"Arrr, me hearties! What be art, ye ask? Art be a fine treasure crafted by the hands of a creative soul. It be a form o' expression, a way to share the beauty and wonders o' the world. It be a splash o' colors on a canvas, a melody playin' in yer ear, or a tale spun with words. Art be a look into the depths o' the human spirit, a glimpse into the mysteries o' life. So, me mateys, let yer hearts be filled with art, for it be the treasure that brings joy and meaning to our pirate lives!\"" ] }, "execution_count": 3, @@ -138,7 +138,7 @@ { "data": { "text/plain": [ - "AIMessage(content='', additional_kwargs={'function_call': {'name': 'revise', 'arguments': '{\\n \"notes\": \"The draft is too short and lacks context. Please provide more information or details about the topic you are writing about.\"\\n}'}}, example=False)" + "AIMessage(content='', additional_kwargs={'function_call': {'name': 'revise', 'arguments': '{\\n \"notes\": \"The current draft is too short and lacks any context or detailed information. Please provide a more comprehensive and detailed draft for review.\"\\n}'}}, example=False)" ] }, "execution_count": 5, @@ -299,7 +299,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "id": "83087e85-b526-4730-b51e-9654dd0b8f69", "metadata": {}, "outputs": [], @@ -311,17 +311,17 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 14, "id": "0371a5ac-7194-4cf9-9dd2-56cb3d1f46d6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[]" + "[{'draft': 'Turtles have specific dietary preferences that vary depending on their species. Sea turtles, for example, primarily consume seaweed, jellyfish, and occasionally fish. On the other hand, land turtles, such as tortoises, mainly graze on grass, flowers, and leafy greens. Some turtles even enjoy fruits like berries and melons in addition to their plant-based diet. Insects also make for a crunchy treat that some turtles may indulge in. Therefore, whether they inhabit land or sea, turtles have a diverse range of food options to keep their bodies nourished and satisfied.'}]" ] }, - "execution_count": 16, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -329,6 +329,28 @@ "source": [ "web_researcher.invoke({\"question\": \"What food do turtles eat?\"})" ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "6bc365e5-06d9-49d9-8278-c3b1138ea73c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[{'draft': 'Turtles are fascinating creatures with a diverse appetite. Depending on their species and habitat, turtles consume a variety of foods. Some turtles primarily eat plants such as seaweed, grass, and algae. For instance, the green sea turtle is known to graze on seagrass beds and algae. Other turtles, like the snapping turtle, have a more carnivorous diet, feasting on insects, fish, and small crustaceans. Additionally, there are land-dwelling turtles that enjoy fruits and vegetables in their diet. For example, the box turtle has been observed eating berries and leafy greens. With such a varied diet, turtles keep their bellies full and maintain their overall health.'}],\n", + " [{'draft': 'Revised draft:\\n\\nHello, readers! You may be wondering where bears live. Well, bears are known to inhabit a wide range of lands, from the icy regions of the Arctic to the lush forests of the jungles. They can be found in North America, Europe, Asia, and even some parts of South America. Bears are highly adaptable creatures, capable of surviving in various habitats, including mountains, tundra, and even deserts. They create dens for hibernation and seek shelter in caves, trees, or dense vegetation. So, be observant, friends, as bears may be encountered in unexpected places!'}]]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[*web_researcher.batch([{\"question\": \"What food do turtles eat?\"}, {\"question\": \"Where do bears live?\"}])]" + ] } ], "metadata": { diff --git a/examples/run_over_dataset.py b/examples/run_over_dataset.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/web-research.ipynb b/examples/web-research.ipynb index bda121ac5..040900284 100644 --- a/examples/web-research.ipynb +++ b/examples/web-research.ipynb @@ -121,7 +121,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 7, "id": "901e1f8d-c973-4998-a731-5dab0c147b8c", "metadata": {}, "outputs": [], @@ -133,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 8, "id": "915bec33-d210-4471-b051-859ecba608be", "metadata": {}, "outputs": [], @@ -157,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 9, "id": "edc0def4-d184-4438-9099-e6604ba9ff28", "metadata": {}, "outputs": [], @@ -167,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 10, "id": "23f89611-7b0f-4f01-b9a7-119124e3341d", "metadata": {}, "outputs": [], @@ -184,7 +184,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 11, "id": "2d4a6b51-bd93-47c2-a301-9593a47df7d4", "metadata": {}, "outputs": [], @@ -196,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 12, "id": "11d3b066-7f95-4ad9-82d0-ba64bbf3e3fa", "metadata": {}, "outputs": [], @@ -204,12 +204,12 @@ "web_researcher = PubSub(\n", " processes=(search_actor, summ_actor),\n", " connection=InMemoryPubSubConnection(),\n", - ")" + ").with_config(run_name=\"WebResearcher\")" ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 13, "id": "bc022d51-69f0-4da9-8025-70afdc3cc6a8", "metadata": {}, "outputs": [ @@ -217,16 +217,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "Fetching pages: 100%|#############################################################################################################################################################################################| 4/4 [00:00<00:00, 6.44it/s]\n" + "Fetching pages: 100%|###################################################| 4/4 [00:01<00:00, 2.46it/s]\n" ] }, { "data": { "text/plain": [ - "[{'answer': 'LangSmith is a platform for building production-grade language model applications. It helps trace and evaluate language model applications and intelligent agents, making it easier to move from prototype to production. LangSmith is developed by LangChain, the company behind the open source LangChain framework. More information can be found in the LangSmith documentation.'}]" + "[{'answer': 'LangSmith is a platform that helps developers build production-grade language model applications and allows for efficient development lifecycles, maintenance, and improvement of AI models. It is built by the developers who created LangChain and integrates seamlessly with that library. LangSmith provides features such as tracing runs associated with an active instance and testing and evaluating prompts or answers generated by the language model applications. It aims to address the challenges of building reliable and maintainable language model applications for production. For more information, you can refer to the LangSmith documentation.'}]" ] }, - "execution_count": 45, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 14, "id": "000f4f24-15ba-476f-8a33-d023081b18d2", "metadata": {}, "outputs": [ @@ -245,28 +245,21 @@ "name": "stderr", "output_type": "stream", "text": [ - "Fetching pages: 0%| | 0/4 [00:00 1\u001b[0m \u001b[43mweb_researcher\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbatch\u001b[49m\u001b[43m(\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mwhat is langsmith\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mwhat is llama\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workplace/langchain/libs/langchain/langchain/schema/runnable/base.py:102\u001b[0m, in \u001b[0;36mRunnable.batch\u001b[0;34m(self, inputs, config, max_concurrency)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m [\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minvoke(inputs[\u001b[38;5;241m0\u001b[39m], configs[\u001b[38;5;241m0\u001b[39m])]\n\u001b[1;32m 101\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m ThreadPoolExecutor(max_workers\u001b[38;5;241m=\u001b[39mmax_concurrency) \u001b[38;5;28;01mas\u001b[39;00m executor:\n\u001b[0;32m--> 102\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mlist\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mexecutor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmap\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minvoke\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mconfigs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.1/lib/python3.10/concurrent/futures/_base.py:608\u001b[0m, in \u001b[0;36mExecutor.map..result_iterator\u001b[0;34m()\u001b[0m\n\u001b[1;32m 605\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m fs:\n\u001b[1;32m 606\u001b[0m \u001b[38;5;66;03m# Careful not to keep a reference to the popped future\u001b[39;00m\n\u001b[1;32m 607\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 608\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m \u001b[43mfs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpop\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mresult\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 609\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 610\u001b[0m \u001b[38;5;28;01myield\u001b[39;00m fs\u001b[38;5;241m.\u001b[39mpop()\u001b[38;5;241m.\u001b[39mresult(end_time \u001b[38;5;241m-\u001b[39m time\u001b[38;5;241m.\u001b[39mmonotonic())\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.1/lib/python3.10/concurrent/futures/_base.py:445\u001b[0m, in \u001b[0;36mFuture.result\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m 443\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m CancelledError()\n\u001b[1;32m 444\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_state \u001b[38;5;241m==\u001b[39m FINISHED:\n\u001b[0;32m--> 445\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__get_result\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 446\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 447\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m()\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.1/lib/python3.10/concurrent/futures/_base.py:390\u001b[0m, in \u001b[0;36mFuture.__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 388\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception:\n\u001b[1;32m 389\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 390\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_exception\n\u001b[1;32m 391\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 392\u001b[0m \u001b[38;5;66;03m# Break a reference cycle with the exception in self._exception\u001b[39;00m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;28mself\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n", - "File \u001b[0;32m~/.pyenv/versions/3.10.1/lib/python3.10/concurrent/futures/thread.py:58\u001b[0m, in \u001b[0;36m_WorkItem.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 58\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m exc:\n\u001b[1;32m 60\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfuture\u001b[38;5;241m.\u001b[39mset_exception(exc)\n", - "File \u001b[0;32m~/workplace/permchain/permchain/pubsub.py:67\u001b[0m, in \u001b[0;36mPubSub.invoke\u001b[0;34m(self, input, config)\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minvoke\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Any, config: Optional[RunnableConfig] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[1;32m 66\u001b[0m collected \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 67\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m chunk \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstream(\u001b[38;5;28minput\u001b[39m, config):\n\u001b[1;32m 68\u001b[0m collected\u001b[38;5;241m.\u001b[39mappend(chunk)\n\u001b[1;32m 69\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m collected\n", - "File \u001b[0;32m~/workplace/permchain/permchain/pubsub.py:176\u001b[0m, in \u001b[0;36mPubSub.stream\u001b[0;34m(self, input, config, max_concurrency)\u001b[0m\n\u001b[1;32m 174\u001b[0m final_output \u001b[38;5;241m=\u001b[39m chunk\n\u001b[1;32m 175\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 176\u001b[0m final_output \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m chunk\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m 178\u001b[0m \u001b[38;5;66;03m# Cleanup\u001b[39;00m\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m fut \u001b[38;5;129;01min\u001b[39;00m inflight:\n", - "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +=: 'dict' and 'dict'" - ] + "data": { + "text/plain": [ + "[[{'answer': 'Based on the search results, LangSmith is a platform that helps developers build and evaluate language model applications. It is designed to assist in moving from prototyping to production and aims to address the challenges of building and maintaining reliable and consistent language model applications. LangSmith is built by the creators of LangChain, a popular language model software tool. It provides features for tracing, testing, evaluating, and monitoring language model applications. LangSmith integrates seamlessly with LangChain and requires a sign-up and API key to access its capabilities.'}],\n", + " [{'answer': 'According to the search results, a llama is a domesticated livestock species that is a descendant of the guanaco and belongs to the camel family. Llamas are primarily used as pack animals and a source of food, wool, hides, tallow for candles, and dried dung for fuel. They are found primarily in South American countries such as Bolivia, Peru, Colombia, Ecuador, Chile, and Argentina. Llamas are known for their long necks, long legs, small heads, large pointed ears, and ability to graze on grass and other plants. They are gregarious animals and can interbreed with other lamoids, such as guanacos, vicuñas, and alpacas. Llama fleece is sheared every two years and consists of coarse guard hairs and short crimped fibers. The fleece is used for knitwear, woven fabrics, rugs, rope, and fabric.'}]]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -283,7 +276,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 15, "id": "43ca019d-a500-4c77-8e62-a46e54ffae7d", "metadata": {}, "outputs": [], @@ -293,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 16, "id": "858a82ae-a73f-4da2-9210-298af789ea30", "metadata": {}, "outputs": [], @@ -327,7 +320,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 17, "id": "c0298fdc-0e7c-4e79-9cb7-cdd4d50fe88f", "metadata": {}, "outputs": [ @@ -336,12 +329,12 @@ "text/plain": [ "['What is the purpose of Langsmith?',\n", " 'Who developed Langsmith?',\n", - " 'What are the features of Langsmith?',\n", - " 'How does Langsmith work?',\n", - " 'Are there any alternatives to Langsmith?']" + " 'What are the key features of Langsmith?',\n", + " 'Are there any alternatives to Langsmith?',\n", + " 'What are the reviews or feedback on Langsmith?']" ] }, - "execution_count": 33, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -352,7 +345,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 18, "id": "a0101bd0-cd95-4b13-ab26-d5d34d703bf9", "metadata": {}, "outputs": [], @@ -376,7 +369,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 19, "id": "d7715e0c-23c0-4985-97a7-bf5b151cf734", "metadata": {}, "outputs": [], @@ -387,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 20, "id": "f274027a-c9f0-4efa-b798-1921b9b376d9", "metadata": {}, "outputs": [], @@ -402,7 +395,7 @@ "research_actor = (\n", " research_inbox.subscribe()\n", " | {\n", - " \"research\": lambda x: web_researcher.batch(x),\n", + " \"research\": web_researcher.map(),\n", " # \"research\": lambda x: [web_researcher.invoke({\"question\": i}) for i in x],\n", " \"question\": Topic.IN.current() | itemgetter(\"question\"),\n", " }\n", @@ -413,7 +406,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 21, "id": "789b636d-23ce-44fb-a8e3-fdfbfabe77ff", "metadata": {}, "outputs": [], @@ -421,12 +414,12 @@ "longer_researcher = PubSub(\n", " processes=(subquestion_actor, research_actor, write_actor),\n", " connection=InMemoryPubSubConnection(),\n", - ")" + ").with_config(run_name=\"LongResearcher\")" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 22, "id": "8f713a31-5f60-4d90-9b95-3f42d8fbbddb", "metadata": {}, "outputs": [ @@ -434,42 +427,40 @@ "name": "stderr", "output_type": "stream", "text": [ - "Fetching pages: 0%| | 0/4 [00:00._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo-16k in organization org-i0zjYONU3PemzJ222esBaAzZ on tokens per min. Limit: 180000 / min. Current: 173743 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n", - "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.._completion_with_retry in 4.0 seconds as it raised RateLimitError: Rate limit reached for default-gpt-3.5-turbo-16k in organization org-i0zjYONU3PemzJ222esBaAzZ on tokens per min. Limit: 180000 / min. Current: 161254 / min. Contact us through our help center at help.openai.com if you continue to have issues..\n" + "Fetching pages: 100%|###################################################| 4/4 [00:01<00:00, 2.33it/s]\u001b[A\u001b[A\n", + "Fetching pages: 100%|###################################################| 4/4 [00:02<00:00, 1.94it/s]\n" ] }, { "data": { "text/plain": [ - "['Research Report: Understanding LangSmith\\n\\nIntroduction:\\nThe purpose of this research report is to provide a comprehensive understanding of LangSmith, a developer platform designed to facilitate the development and management of Language Model applications (LLMs). Through an analysis of the gathered research, this report aims to answer the question: \"What is LangSmith?\"\\n\\nResearch Findings:\\n\\n1. LangSmith Overview:\\nLangSmith is a unified platform that helps developers trace, evaluate, and monitor LLM applications and intelligent agents. It aims to simplify the process of moving from prototype to production by providing features such as tracing runs, testing prompts or answers, and exporting datasets and runs for further analysis. LangSmith offers comprehensive visibility into the chain sequence of calls, real-time insights, and observability features to monitor LLM applications. It emphasizes best practices and offers useful tools and resources for developers working with LLMs.\\n\\n2. Key Features of LangSmith:\\nLangSmith offers several key features to assist developers in building and managing LLM applications. These include:\\n- Tracing and evaluating the behavior of LLM applications.\\n- Debugging and experimentation capabilities.\\n- Sharing work with others.\\n- Creating datasets for testing and evaluation.\\n- Evaluating models based on created datasets.\\n- Monitoring the behavior and performance of LLM applications.\\n- Comprehensive visibility into the entire chain sequence of calls.\\n\\n3. LangSmith\\'s Integration with LangChain:\\nLangSmith seamlessly integrates with LangChain, a library for prototyping LLM applications. This integration allows developers to leverage the composability of LangChain and build applications with large language models effectively. LangChain supports features such as memory, custom datasets, and more.\\n\\n4. Potential Alternatives to LangSmith:\\nBased on the research findings, several potential alternatives to LangSmith have been identified. These include:\\n- LangChain: An open-source framework for building applications with large language models through composability.\\n- GradientJ: A platform to build, orchestrate, and manage complex LLM applications at scale.\\n- LLMOps.Space: A community and resource hub focused on deploying LLMs into production.\\n- Vellum: A development platform aimed at production LLM applications, providing tools for monitoring, version control, and testing datasets.\\n- Llama2: An open-source large language model from Meta that can be fine-tuned and deployed.\\n- Openlayer: A platform focused on ML model testing, monitoring, and improvement.\\n- Backengine: A platform that allows creating and deploying backend APIs using natural language descriptions.\\n- QueryVary: A platform for systematically designing and refining prompts for LLMs.\\n\\nConclusion:\\nIn conclusion, LangSmith is a developer platform designed to simplify the development and management of Language Model applications (LLMs). It provides developers with tools for tracing, testing, evaluating, and monitoring LLM applications, along with comprehensive visibility and real-time insights. LangSmith aims to empower developers and handle the complexity of LLM applications effectively. By integrating seamlessly with LangChain, it offers enhanced capabilities for building applications with large language models. While LangSmith is a prominent platform, developers may also consider other alternatives such as LangChain, GradientJ, LLMOps.Space, Vellum, and more, depending on their specific requirements.']" + "[\"Research Report: Understanding LangSmith - A Unified Platform for Building LLM Applications\\n\\nIntroduction:\\nThe purpose of this research report is to provide a comprehensive understanding of LangSmith, a unified platform designed to assist developers in building production-grade Language Model applications (LLMs). By exploring various sub-questions, we have gathered information about LangSmith's features, development team, alternatives, and user feedback.\\n\\n1. What is LangSmith?\\nLangSmith is a unified platform developed by LangChain, the creators of LangChain itself. It aims to streamline the development lifecycle, maintenance, and improvement of LLM applications. By providing tools for debugging, testing, evaluating, and monitoring LLM applications, LangSmith enables developers to transition from prototype to production seamlessly. It addresses the challenges associated with building reliable and maintainable LLM applications in production environments.\\n\\n2. Features and Integration:\\nLangSmith offers a range of features to facilitate the development and management of LLM applications. These include tracing runs, testing and evaluating LLM-generated prompts or answers, and visualizing and replaying traces. The platform integrates seamlessly with LangChain, providing native integration for debugging and monitoring LLM applications.\\n\\n3. Getting Started with LangSmith:\\nTo utilize LangSmith, users need to sign up for an account and create an API key. Additionally, having a GitHub repository and an OpenAI API key is necessary for integration. Although LangSmith is currently in beta, periodic access to new sign-ups is available.\\n\\n4. Alternatives to LangSmith:\\nBased on our research, we have identified several alternatives to LangSmith for developers working with LLM applications. These alternatives offer similar functionalities and may serve as viable options depending on specific project requirements. Some notable alternatives include:\\n\\n- LangChain: An open source framework for building conversational AI agents, which integrates with LangSmith for debugging and monitoring.\\n- Autoblocks: A tool that monitors and improves AI models powered by large language models, providing an SDK for easy integration.\\n- BenchLLM: An open source tool specifically designed for evaluating large language models, supporting models from various providers including OpenAI and LangChain.\\n- GradientJ: A comprehensive platform for building, evaluating, and managing large language models, offering features such as prompt chaining and data integration.\\n- Portkey: An LMOps platform catering to the deployment of production-ready LLM applications, offering model management, monitoring, and versioning tools.\\n- Vellum AI: A platform equipped with tools for developing LLM applications, including prompt engineering, version control, testing, and monitoring. Compatible with multiple LLM providers.\\n- Openlayer: A collaborative platform that facilitates aligning expectations around LLM quality and performance, providing diagnostic tools for issue resolution and iteration.\\n- Metal: A tool that focuses on providing LLM developers with a seamless workflow, although specific details about its features are not available.\\n\\n5. User Feedback:\\nLangSmith has received positive reviews and feedback from the community. Although the reviews were based on the tool's description and potential rather than personal usage, LangSmith has an overall rating of 5/5. Users have praised the platform for simplifying the transition of projects from the prototyping phase to full-fledged production. LangSmith's ability to manage the complexity of LLM applications and enhance product development and iteration capabilities has been highlighted.\\n\\nConclusion:\\nIn conclusion, LangSmith is a unified platform developed by LangChain to assist developers in building production-grade Language Model applications. It offers a range of features for debugging, testing, evaluating, and monitoring LLM applications, aiming to simplify the development lifecycle, maintenance, and improvement of these applications. Although alternatives exist, LangSmith has received positive feedback and has the potential to enhance productivity and workflow for developers working with LLMs.\"]" ] }, - "execution_count": 38, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -503,7 +494,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.1" + "version": "3.11.4" } }, "nbformat": 4, diff --git a/permchain/pubsub.py b/permchain/pubsub.py index e495a9bb3..e066a86f2 100644 --- a/permchain/pubsub.py +++ b/permchain/pubsub.py @@ -15,17 +15,12 @@ from typing import ( Set, Tuple, TypeVar, - Union, ) -from langchain.load.dump import dumpd +from langchain.callbacks.manager import CallbackManagerForChainRun from langchain.load.serializable import Serializable from langchain.schema.runnable import Runnable, RunnableConfig, patch_config -from langchain.schema.runnable.config import ( - ensure_config, - get_callback_manager_for_config, - get_executor_for_config, -) +from langchain.schema.runnable.config import get_executor_for_config from permchain.connection import PubSubConnection from permchain.constants import CONFIG_GET_KEY, CONFIG_SEND_KEY @@ -67,38 +62,22 @@ class PubSub(Serializable, Runnable[Any, Any], ABC): class Config: arbitrary_types_allowed = True - def invoke(self, input: Any, config: Optional[RunnableConfig] = None) -> Any: - collected = [] - for chunk in self.stream(input, config): - collected.append(chunk) - return collected - - def batch( + def _transform( self, - input: Sequence[Any], - config: Optional[Union[RunnableConfig, List[RunnableConfig]]] = None, - ) -> Any: - configs = self._get_config_list(config, len(input)) - with get_executor_for_config(configs[0]) as executor: - return super().batch( - input, - [patch_config(config, executor=executor) for config in configs], - ) - - def stream( - self, - input: Any, - config: Optional[RunnableConfig] = None, + input: Iterator[Any], + run_manager: CallbackManagerForChainRun, + config: RunnableConfig, ) -> Iterator[Any]: input_processes, listener_processes = partition( lambda r: r.topic.name == INPUT_TOPIC, self.processes ) - # setup callbacks - config = ensure_config(config) - callback_manager = get_callback_manager_for_config(config) - # start the root run - run_manager = callback_manager.on_chain_start(dumpd(self), {"input": input}) + input_value = None + for chunk in input: + if input_value is None: + input_value = chunk + else: + input_value += chunk with get_executor_for_config(config) as executor: # Track inflight futures @@ -143,7 +122,7 @@ class PubSub(Serializable, Runnable[Any, Any], ABC): def get(topic_name: str) -> Any: if topic_name == INPUT_TOPIC: - return input + return input_value elif topic_name == process.topic.name: return value else: @@ -158,8 +137,8 @@ class PubSub(Serializable, Runnable[Any, Any], ABC): config={ **patch_config( config, - callbacks=run_manager.get_child(process.topic.name), - executor=executor, + callbacks=run_manager.get_child(), + run_name=f"Topic: {process.topic.name}", ), CONFIG_SEND_KEY: send, CONFIG_GET_KEY: get, @@ -176,37 +155,42 @@ class PubSub(Serializable, Runnable[Any, Any], ABC): prefix_topic_name(process.topic.name), partial(run_once, process) ) - # Run input processes once + # Send input to each input process for process in input_processes: - run_once(process, input) + run_once(process, input_value) try: # Yield output until all processes are done - final_output = None for chunk in output: yield chunk - if final_output is None: - final_output = chunk - else: - final_output += chunk finally: - # Cleanup + # Disconnect from all topics for process in listener_processes: self.connection.disconnect(prefix_topic_name(process.topic.name)) + # Cancel all inflight futures while inflight: inflight.pop().cancel() # Raise exceptions if any if exceptions: - run_manager.on_chain_error(exceptions[0]) raise exceptions[0] - else: - run_manager.on_chain_end( - final_output - if isinstance(final_output, dict) - else {"output": final_output} - ) + + def stream( + self, + input: Any, + config: Optional[RunnableConfig] = None, + **kwargs: Optional[Any], + ) -> Iterator[Any]: + yield from self._transform_stream_with_config( + iter([input]), self._transform, config, **kwargs + ) + + def invoke(self, input: Any, config: Optional[RunnableConfig] = None) -> Any: + collected = [] + for chunk in self.stream(input, config): + collected.append(chunk) + return collected PubSub.update_forward_refs() diff --git a/permchain/topic.py b/permchain/topic.py index e3e01fbb3..38e327d20 100644 --- a/permchain/topic.py +++ b/permchain/topic.py @@ -94,14 +94,14 @@ class RunnableSubscriber(RunnableBinding[T, Any]): raise NotImplementedError() -class RunnablePublisher(RunnablePassthrough[T]): +class RunnablePublisher(Runnable[T, T]): topic: Topic[T] def invoke(self, input: T, config: Optional[RunnableConfigForPubSub] = None) -> T: send = config.get(CONFIG_SEND_KEY, None) if send is not None: send(self.topic.name, input) - return super().invoke(input, config) + return input class RunnablePublisherEach(RunnablePublisher[Sequence[T]]):