diff --git a/README.md b/README.md index 87aab4f13..ad8725a89 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,6 @@ Optionally, we can set up [LangSmith](https://docs.smith.langchain.com/) for bes ```shell export LANGCHAIN_TRACING_V2="true" export LANGCHAIN_API_KEY=ls__... -export LANGCHAIN_ENDPOINT=https://api.langchain.plus ``` ### Set up the tools @@ -426,41 +425,41 @@ All agent state is represented as a list of messages. This specifically uses OpenAI function calling. This is recommended agent executor for newer chat based models that support function calling. -- [Getting Started Notebook](examples/chat_agent_executor_with_function_calling/base.ipynb): Walks through creating this type of executor from scratch -- [High Level Entrypoint](examples/chat_agent_executor_with_function_calling/high-level.ipynb): Walks through how to use the high level entrypoint for the chat agent executor. +- [Getting Started Notebook](https://github.com/langchain-ai/langgraph/blob/main/examples/chat_agent_executor_with_function_calling/base.ipynb): Walks through creating this type of executor from scratch +- [High Level Entrypoint](https://github.com/langchain-ai/langgraph/blob/main/examples/chat_agent_executor_with_function_calling/high-level.ipynb): Walks through how to use the high level entrypoint for the chat agent executor. **Modifications** We also have a lot of examples highlighting how to slightly modify the base chat agent executor. These all build off the [getting started notebook](examples/chat_agent_executor_with_function_calling/base.ipynb) so it is recommended you start with that first. -- [Human-in-the-loop](examples/chat_agent_executor_with_function_calling/human-in-the-loop.ipynb): How to add a human-in-the-loop component -- [Force calling a tool first](examples/chat_agent_executor_with_function_calling/force-calling-a-tool-first.ipynb): How to always call a specific tool first -- [Respond in a specific format](examples/chat_agent_executor_with_function_calling/respond-in-format.ipynb): How to force the agent to respond in a specific format -- [Dynamically returning tool output directly](examples/chat_agent_executor_with_function_calling/dynamically-returning-directly.ipynb): How to dynamically let the agent choose whether to return the result of a tool directly to the user -- [Managing agent steps](examples/chat_agent_executor_with_function_calling/managing-agent-steps.ipynb): How to more explicitly manage intermediate steps that an agent takes +- [Human-in-the-loop](https://github.com/langchain-ai/langgraph/blob/main/examples/chat_agent_executor_with_function_calling/human-in-the-loop.ipynb): How to add a human-in-the-loop component +- [Force calling a tool first](https://github.com/langchain-ai/langgraph/blob/main/examples/chat_agent_executor_with_function_calling/force-calling-a-tool-first.ipynb): How to always call a specific tool first +- [Respond in a specific format](https://github.com/langchain-ai/langgraph/blob/main/examples/chat_agent_executor_with_function_calling/respond-in-format.ipynb): How to force the agent to respond in a specific format +- [Dynamically returning tool output directly](https://github.com/langchain-ai/langgraph/blob/main/examples/chat_agent_executor_with_function_calling/dynamically-returning-directly.ipynb): How to dynamically let the agent choose whether to return the result of a tool directly to the user +- [Managing agent steps](https://github.com/langchain-ai/langgraph/blob/main/examples/chat_agent_executor_with_function_calling/managing-agent-steps.ipynb): How to more explicitly manage intermediate steps that an agent takes ### AgentExecutor This agent executor uses existing LangChain agents. -- [Getting Started Notebook](examples/agent_executor/base.ipynb): Walks through creating this type of executor from scratch -- [High Level Entrypoint](examples/agent_executor/high-level.ipynb): Walks through how to use the high level entrypoint for the chat agent executor. +- [Getting Started Notebook](https://github.com/langchain-ai/langgraph/blob/main/examples/agent_executor/base.ipynb): Walks through creating this type of executor from scratch +- [High Level Entrypoint](https://github.com/langchain-ai/langgraph/blob/main/examples/agent_executor/high-level.ipynb): Walks through how to use the high level entrypoint for the chat agent executor. **Modifications** We also have a lot of examples highlighting how to slightly modify the base chat agent executor. These all build off the [getting started notebook](examples/agent_executor/base.ipynb) so it is recommended you start with that first. -- [Human-in-the-loop](examples/agent_executor/human-in-the-loop.ipynb): How to add a human-in-the-loop component -- [Force calling a tool first](examples/agent_executor/force-calling-a-tool-first.ipynb): How to always call a specific tool first -- [Managing agent steps](examples/agent_executor/managing-agent-steps.ipynb): How to more explicitly manage intermediate steps that an agent takes +- [Human-in-the-loop](https://github.com/langchain-ai/langgraph/blob/main/examples/agent_executor/human-in-the-loop.ipynb): How to add a human-in-the-loop component +- [Force calling a tool first](https://github.com/langchain-ai/langgraph/blob/main/examples/agent_executor/force-calling-a-tool-first.ipynb): How to always call a specific tool first +- [Managing agent steps](https://github.com/langchain-ai/langgraph/blob/main/examples/agent_executor/managing-agent-steps.ipynb): How to more explicitly manage intermediate steps that an agent takes ### Async If you are running LangGraph in async workflows, you may want to create the nodes to be async by default. -In order for a walkthrough on how to do that, see [this documentation](examples/async.ipynb) +In order for a walkthrough on how to do that, see [this documentation](https://github.com/langchain-ai/langgraph/blob/main/examples/async.ipynb) ### Streaming Tokens Sometimes language models take a while to respond and you may want to stream tokens to end users. -For a guide on how to do this, see [this documentation](examples/streaming-tokens.ipynb) +For a guide on how to do this, see [this documentation](https://github.com/langchain-ai/langgraph/blob/main/examples/streaming-tokens.ipynb) ## Documentation diff --git a/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb b/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb new file mode 100644 index 000000000..71ddb19a1 --- /dev/null +++ b/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb @@ -0,0 +1,409 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a3e3ebc4-57af-4fe4-bdd3-36aff67bf276", + "metadata": {}, + "source": [ + "# Chat Bot Evaluation as Multi-agent Simulation\n", + "\n", + "When building a chat bot, such as a customer support assistant, it can be hard to properly evalute your bot's performance. It's time-consuming to have to manually interact with it intensively for each code change.\n", + "\n", + "One way to make the evaluation process easier and more reproducible is to simulate a user interaction.\n", + "\n", + "With LangGraph, it's easy to set this up. Below is an example of how to create a \"virtual user\" to simulate a conversation.\n", + "\n", + "The overall simulation looks something like this:\n", + "\n", + "![diagram](./img/virtual_user_diagram.png)\n", + "\n", + "First, we'll set up our environment." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "0d30b6f7-3bec-4d9f-af50-43dfdc81ae6c", + "metadata": {}, + "outputs": [], + "source": [ + "# %%capture --no-stderr\n", + "# %pip install -U langgraph langchain langchain_openai" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "30c2f3de-c730-4aec-85a6-af2c2f058803", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "import uuid\n", + "\n", + "\n", + "def _set_if_undefined(var: str):\n", + " if not os.environ.get(var):\n", + " os.environ[var] = getpass(f\"Please provide your {var}\")\n", + "\n", + "\n", + "_set_if_undefined(\"OPENAI_API_KEY\")\n", + "_set_if_undefined(\"LANGCHAIN_API_KEY\")\n", + "\n", + "# Optional, add tracing in LangSmith.\n", + "# This will help you visualize and debug the control flow\n", + "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n", + "os.environ[\"LANGCHAIN_PROJECT\"] = \"Agent Simulation Evaluation\"" + ] + }, + { + "cell_type": "markdown", + "id": "6ef4528d-6b2a-47c7-98b5-50f14984a304", + "metadata": {}, + "source": [ + "## 1. Define Chat Bot\n", + "\n", + "Next, we will define our chat bot. For this notebook, we assume the bot's API accepts a list of messages and responds with a message. If you want to update this, all you'll have to change is this section and the \"get_messages_for_agent\" function in \n", + "the simulator below.\n", + "\n", + "The implementation within `my_chat_bot` is configurable and can even be run on another system (e.g., if your system isn't running in python)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "828479af-cf9c-4888-a365-599643a96b55", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import List\n", + "\n", + "import openai\n", + "\n", + "\n", + "# This is flexible, but you can define your agent here, or call your agent API here.\n", + "def my_chat_bot(messages: List[dict]) -> dict:\n", + " completion = openai.chat.completions.create(\n", + " messages=messages, model=\"gpt-3.5-turbo\"\n", + " )\n", + " return completion.choices[0].message.model_dump()" + ] + }, + { + "cell_type": "markdown", + "id": "321312b4-a1f0-4454-a481-fdac4e37cb7d", + "metadata": {}, + "source": [ + "## 2. Define the Agent Simulation\n", + "\n", + "The code below creates a LangGraph workflow to run the simulation. The main components are:\n", + "\n", + "1. Simulation state: the inputs to each node in the graph, containing the messages (stateful) and the details about the simulated user.\n", + "2. Functions to interoperate between the simulation state and your chat bot's API\n", + "3. The simulated user definition.\n", + "4. The graph itself, with a conditional stopping criterion.\n", + "\n", + "Read the comments in the code below for more information." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cc93d114-fa65-4021-a67e-b1e6d4edc88a", + "metadata": {}, + "outputs": [], + "source": [ + "import operator\n", + "from typing import Annotated, Callable, Dict, List, TypedDict\n", + "\n", + "from langchain.adapters.openai import convert_message_to_dict\n", + "from langchain_core.messages import AIMessage, BaseMessage, HumanMessage\n", + "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n", + "from langchain_core.runnables import chain\n", + "from langchain_openai import ChatOpenAI\n", + "\n", + "from langgraph.graph import END, StateGraph\n", + "\n", + "SIMULATED_USER_NAME = \"simulated\"\n", + "\n", + "\n", + "# This is just an example, we can\n", + "# configure additional parameters if\n", + "# you want more control\n", + "class SimulatedUserConfig(TypedDict):\n", + " system_prompt: str\n", + "\n", + "\n", + "# This is the input to every node in the simulation graph\n", + "# It tracks the graph state over time. Our only \"state\"\n", + "# is the conversation messages, while the user config\n", + "# is provided to make the virtual user more unique or realistic\n", + "class Environment(TypedDict):\n", + " messages: Annotated[List[BaseMessage], operator.add]\n", + " simulated_user_config: SimulatedUserConfig\n", + "\n", + "\n", + "# We currently let the virtual user decide if the conversation can end\n", + "# we could also track max conversation turns, add a conversation \"supervisor\"\n", + "# or use other heuristics to control the dialogue flow\n", + "def should_continue(state: Environment):\n", + " \"\"\"Determine if the simulation should continue.\"\"\"\n", + " if state[\"messages\"][-1].content.strip().endswith(\"FINISHED\"):\n", + " return \"end\"\n", + " return \"continue\"\n", + "\n", + "\n", + "## The next two functions define the API between the simulation\n", + "# and the chat bot you wish to test.\n", + "# We are assuming your chat bot accepts a list of OAI messages\n", + "@chain\n", + "def get_messages_for_agent(state: Environment):\n", + " \"\"\"Convert the simulation state to the input\n", + "\n", + " for your agent you want to evaluate.\"\"\"\n", + " return [convert_message_to_dict(message) for message in state[\"messages\"]]\n", + "\n", + "\n", + "# This takes the output of your chat bot\n", + "# and adds it to the simulation state\n", + "def get_response_message_from_agent(agent_output):\n", + " \"\"\"Get the response from the agent you are evaluting,\n", + " and use it to update the simulation state.\"\"\"\n", + " # If we do an ai message here, the user proxy llm\n", + " # will usually forget it's acting.\n", + " return {\"messages\": [HumanMessage(content=agent_output[\"content\"])]}\n", + "\n", + "\n", + "# This is run once at the beginning of the simulation.\n", + "# It's more convenient to just write an input string\n", + "# than to pass in a full message, but this could be removed below\n", + "def enter(inputs: dict):\n", + " \"\"\"Start the simulation. This makes it less verbose to invoke.\"\"\"\n", + " inputs[\"messages\"] = [\n", + " HumanMessage(content=inputs[\"input\"], name=SIMULATED_USER_NAME)\n", + " ]\n", + " return inputs\n", + "\n", + "\n", + "def create_simulation(chat_bot: Callable[[List[Dict]], Dict], simulated_user_llm=None):\n", + " \"\"\"Create a chat bot simulation graph.\n", + "\n", + " Args:\n", + " - chat_bot: the agent you are evaluating. Accepts a list of openai messages\n", + " and returns an openai assistant message\n", + " - simulated_user_llm: the LLM to power your virtual user.\n", + " Defaults to gpt-4-1106-preview\n", + " Returns:\n", + " - simulation: an runnable object formed from compiling the state graph\n", + " \"\"\"\n", + " # This defines the virtual user proxy\n", + " prompt = ChatPromptTemplate.from_messages(\n", + " [\n", + " (\n", + " \"system\",\n", + " \"You are role-playing a human character: '{name}'. \"\n", + " \"You are not an AI assistant and you are not supposed to help or assist.\"\n", + " \" You must behave as this human would throughout the conversation below.\\n\\n\"\n", + " \"Your messages will bear the name 'simulated', but DO NOT under any circumstances\"\n", + " \"say that you are 'simulated'. You will be evaluated based on how realistic your\"\n", + " \"impersonation of this character is. This must feel real! Here are the details for your character:\"\n", + " \"\\n\"\n", + " \"{system_prompt}\" # This is the value you provide to characterize the user\n", + " '\\n\\nWhen you are finished with the conversation, respond with a single word \"FINISHED\"',\n", + " ),\n", + " MessagesPlaceholder(variable_name=\"messages\"),\n", + " ]\n", + " ).partial(name=SIMULATED_USER_NAME)\n", + " simulated_user_llm = simulated_user_llm or ChatOpenAI(model=\"gpt-4-1106-preview\")\n", + " user_proxy = (\n", + " (lambda x: {**x, **x[\"simulated_user_config\"]})\n", + " | prompt\n", + " | simulated_user_llm\n", + " | (\n", + " lambda x: {\n", + " \"messages\": [HumanMessage(content=x.content, name=SIMULATED_USER_NAME)]\n", + " }\n", + " )\n", + " )\n", + " graph_builder = StateGraph(Environment)\n", + " graph_builder.add_node(\"user\", user_proxy)\n", + " graph_builder.add_node(\n", + " # The \"|\" syntax composes these steps in the pipeline to map between\n", + " # the simulation state and your chat bot's API\n", + " \"chat_bot\",\n", + " get_messages_for_agent | chat_bot | get_response_message_from_agent,\n", + " )\n", + " # Every response from your chat bot will automatically go to the\n", + " # simulated user\n", + " graph_builder.add_edge(\"chat_bot\", \"user\")\n", + " graph_builder.add_conditional_edges(\n", + " \"user\",\n", + " should_continue,\n", + " # If the finish criteria are met, we will stop the simulation,\n", + " # otherwise, the virtual user's message will be sent to your chat bot\n", + " {\n", + " \"end\": END,\n", + " \"continue\": \"chat_bot\",\n", + " },\n", + " )\n", + " # The input will first go to your chat bot\n", + " graph_builder.set_entry_point(\"chat_bot\")\n", + " return (enter | graph_builder.compile()).with_config(run_name=\"Agent Simulation\")" + ] + }, + { + "cell_type": "markdown", + "id": "2e0bd26e-8c1d-471d-9fef-d95dc0163491", + "metadata": {}, + "source": [ + "## 3. Run Simulation\n", + "\n", + "Now we can evaluate our chat bot! We will provide information about the simulated user (as a system prompt)\n", + "as well as the initial input message from that simulated user to the chat bot." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "4d495ccc-0f5f-4194-89e5-15dccbbb7412", + "metadata": {}, + "outputs": [], + "source": [ + "simulation = create_simulation(my_chat_bot)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "fa91e733-3493-43c2-ba21-171cf78deef8", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Skipping write for channel input which has no readers\n" + ] + } + ], + "source": [ + "from langchain_core.tracers.context import tracing_v2_enabled\n", + "\n", + "# The tracing context manager lets us easily fetch the trace URL in-context.\n", + "# You can turn this off if you don't want to trace the execution.\n", + "with tracing_v2_enabled() as tracer:\n", + " result = simulation.invoke(\n", + " {\n", + " \"simulated_user_config\": {\n", + " \"system_prompt\": \"You are on a budget. Your family is hard to please.\"\n", + " \" They all like the beach, except for Aunt Lily, who prefers the mountains.\"\n", + " },\n", + " \"input\": \"help me plan my family vacation\",\n", + " }\n", + " )\n", + " # You can go to this run to review the entire simulation trace\n", + " url = tracer.get_run_url()" + ] + }, + { + "cell_type": "markdown", + "id": "73ff30e4-1992-4bc9-834d-f4c08b281d20", + "metadata": {}, + "source": [ + "## (Optional) Review Results\n", + "\n", + "If you've traced the run, you can see the full simulation trace in the UI by clicking on the url.\n", + "Select the last 'ChatOpenAI' call in the trace to see the full conversation in a single view.\n", + "\n", + "![full-conversation](./img/virtual_user_full_convo.png)\n", + "\n", + "\n", + "From this run, you can manually annotate it to score its quality. This feedback can be used to compare the quality of different versions of your chat bot.\n", + "\n", + "![annotate](./img/virtual_user_annotate.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "702d008c-dac5-478f-8239-ede3050573c6", + "metadata": {}, + "outputs": [], + "source": [ + "url" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "0de1664f-1625-42e7-9dad-eb2c061b88d4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[HumanMessage(content='help me plan my family vacation', name='simulated'),\n", + " HumanMessage(content=\"Sure! I'd be happy to help you plan your family vacation. Can you provide more details about your preferences, such as the destination, budget, duration of the trip, and any specific activities or attractions you have in mind?\"),\n", + " HumanMessage(content=\"Oh, planning family vacations is always a bit of a juggling act, isn't it? We've got a variety of tastes in my family too, so I totally get where you're coming from. We're on a budget, so we usually look for places that won't break the bank. Everyone loves the beach—it's just Aunt Lily who's the odd one out, preferring the mountains.\\n\\nHere's a thought, maybe you can find a coastal area that's near some mountains? That way, the majority of the family gets to enjoy the sand and surf while Aunt Lily isn't too far from a mountain getaway. Depending on where you live, there might be some places not too far away that offer both. \\n\\nFor instance, places like the Central Coast of California have beaches and they're not too far from mountains. Or you could look into a spot like the South of France, if you're up for international travel and can find some deals. I've also heard that places like Costa Rica have both, but I've never been there myself.\\n\\nAs for the budget, I'm always on the lookout for off-season deals or vacation rentals that can accommodate the whole family. It can be way more cost-effective than booking multiple hotel rooms, and you can save a bit by cooking meals at the rental rather than eating out all the time.\\n\\nHave you thought about any specific destinations yet?\", name='simulated'),\n", + " HumanMessage(content=\"Those are great suggestions! Finding a destination that offers both beach and mountain options can be a great compromise for your family. Here are a few more specific destination ideas that might fit your preferences:\\n\\n1. The Oregon Coast, USA: Known for its stunning coastline and nearby mountain ranges like the Cascade Range, the Oregon Coast offers a mix of beautiful beaches, charming coastal towns, and opportunities for hiking in the mountains.\\n\\n2. Bali, Indonesia: This tropical island destination offers gorgeous beaches as well as volcanic mountains like Mount Batur. You can relax on the beach, explore temples, try water sports, and even trek through rice terraces and lush forests.\\n\\n3. Split, Croatia: Located on the stunning Dalmatian Coast, Split offers a mix of beach relaxation and nearby mountain hiking opportunities in places like the Biokovo nature park. Plus, you can explore the historic Old Town and nearby islands like Hvar.\\n\\n4. Cape Town, South Africa: With its iconic Table Mountain and beautiful Atlantic beaches like Camps Bay, Cape Town provides the best of both worlds. You can take a cable car up Table Mountain, visit the penguins at Boulders Beach, and even go on a wine tour in the nearby Cape Winelands.\\n\\nWhen it comes to budget-friendly options, consider booking vacation rentals, researching affordable or all-inclusive resorts, and keeping an eye out for discounts on flights and attractions. It's also advisable to be flexible with your travel dates, as traveling during the offseason can often result in more affordable prices.\\n\\nLet me know if you need any more information or help with planning specific activities or accommodations in any of these destinations!\"),\n", + " HumanMessage(content=\"Oh, those are some fantastic ideas, really! Each of those spots has something unique to offer. I'll definitely have to look into the Oregon Coast. It has that rugged charm, and I've heard it's not as pricey as California. Bali sounds like a dream, honestly, but I have to admit, international travel might be a bit much for the budget this time around. \\n\\nCroatia is one of those places I've always wanted to visit, with all that beautiful coastline and history, but again, might be a stretch budget-wise. Cape Town would be an adventure for sure, but South Africa is a big trip. It's probably out of our range for now.\\n\\nI really appreciate the suggestions about being flexible with travel dates and looking at vacation rentals. That's the kind of approach we usually take. We try to avoid the peak seasons to save some money and find those hidden deals.\\n\\nIt sounds like you've done a fair bit of traveling yourself, or you're just really good at sniffing out the cool spots to visit. Do you travel a lot?\", name='simulated'),\n", + " HumanMessage(content=\"I'm glad you found the suggestions helpful! The Oregon Coast is definitely a more budget-friendly option compared to some other coastal destinations. It offers stunning landscapes, charming towns, and the opportunity to explore both the beach and the mountains.\\n\\nBali is indeed a dream destination, but it's understandable that international travel might not fit within the budget this time. It's always good to keep it in mind for future trips though, as it offers a unique cultural experience along with beautiful beaches and mountains.\\n\\nCroatia is known for its stunning coastline and historic cities like Split and Dubrovnik, but it can sometimes be on the more expensive side. It's always worth checking for deals and considering different accommodation options to make it more affordable.\\n\\nAnd yes, South Africa and Cape Town are definitely big trips. If it's not feasible for the current vacation, you can always keep it on your bucket list for future adventures when the budget allows.\\n\\nAs for your question, I do love to travel and explore different places whenever I get the chance. I'm also always researching and learning about new destinations to be able to offer suggestions and help others plan their trips. It's a passion of mine! If you ever need more help or have any specific questions about the destinations or planning, feel free to ask.\"),\n", + " HumanMessage(content='FINISHED', name='simulated')]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gch # These are the message from the final simulation state\n", + "result[\"messages\"]" + ] + }, + { + "cell_type": "markdown", + "id": "23db8891-5db3-4a98-a283-53d45cd28c60", + "metadata": {}, + "source": [ + "## Conclusion\n", + "\n", + "In this notebook, you set up a multi-agent simulation to review how your chat bot behaves with simulated users.\n", + "\n", + "To implement this for your chat bot, you can create a dataset of user profiles and questions your chat bot should handle and run periodically. You can use an LLM-as-judge to give the bot an initial score and then manually review to spot check. \n", + "\n", + "LangGraph gives you full control over the simulation so you can manually change the simulated user and the conversation dynamics." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dde4f2b5-cfe8-4ff0-99ea-fe2c5fed70c0", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/advanced_agents/multi-agent/img/virtual_user_annotate.png b/examples/advanced_agents/multi-agent/img/virtual_user_annotate.png new file mode 100644 index 000000000..8886efae4 Binary files /dev/null and b/examples/advanced_agents/multi-agent/img/virtual_user_annotate.png differ diff --git a/examples/advanced_agents/multi-agent/img/virtual_user_diagram.png b/examples/advanced_agents/multi-agent/img/virtual_user_diagram.png new file mode 100644 index 000000000..a0cefb06b Binary files /dev/null and b/examples/advanced_agents/multi-agent/img/virtual_user_diagram.png differ diff --git a/examples/advanced_agents/multi-agent/img/virtual_user_full_convo.png b/examples/advanced_agents/multi-agent/img/virtual_user_full_convo.png new file mode 100644 index 000000000..8da9c2d3c Binary files /dev/null and b/examples/advanced_agents/multi-agent/img/virtual_user_full_convo.png differ diff --git a/langgraph/channels/last_value.py b/langgraph/channels/last_value.py index c585ceb86..858943c91 100644 --- a/langgraph/channels/last_value.py +++ b/langgraph/channels/last_value.py @@ -44,7 +44,7 @@ class LastValue(Generic[Value], BaseChannel[Value, Value, Value]): if len(values) == 0: return if len(values) != 1: - raise InvalidUpdateError() + raise InvalidUpdateError("LastValue can only receive one value per step.") self.value = values[-1] diff --git a/langgraph/checkpoint/__init__.py b/langgraph/checkpoint/__init__.py index 7a52e0c6f..3d8a1ab76 100644 --- a/langgraph/checkpoint/__init__.py +++ b/langgraph/checkpoint/__init__.py @@ -1,7 +1,8 @@ -from langgraph.checkpoint.base import BaseCheckpointSaver, CheckpointAt +from langgraph.checkpoint.base import BaseCheckpointSaver, Checkpoint, CheckpointAt from langgraph.checkpoint.memory import MemorySaver __all__ = [ + "Checkpoint", "CheckpointAt", "BaseCheckpointSaver", "MemorySaver", diff --git a/langgraph/checkpoint/base.py b/langgraph/checkpoint/base.py index e40f3fcfc..699dc9dd8 100644 --- a/langgraph/checkpoint/base.py +++ b/langgraph/checkpoint/base.py @@ -19,13 +19,17 @@ class Checkpoint(TypedDict): versions_seen: defaultdict[str, defaultdict[str, int]] +def _seen_dict(): + return defaultdict(int) + + def empty_checkpoint() -> Checkpoint: return Checkpoint( v=1, ts=datetime.now(timezone.utc).isoformat(), channel_values={}, channel_versions=defaultdict(int), - versions_seen=defaultdict(lambda: defaultdict(int)), + versions_seen=defaultdict(_seen_dict), ) diff --git a/langgraph/graph/state.py b/langgraph/graph/state.py index 45fb5ec65..1a3f5530e 100644 --- a/langgraph/graph/state.py +++ b/langgraph/graph/state.py @@ -122,3 +122,4 @@ def _is_field_binop(typ: Type[Any]) -> Optional[BinaryOperatorAggregate]: ] ): return BinaryOperatorAggregate(typ, meta[0]) + return None diff --git a/langgraph/pregel/__init__.py b/langgraph/pregel/__init__.py index b557b9c5d..115e0cbf6 100644 --- a/langgraph/pregel/__init__.py +++ b/langgraph/pregel/__init__.py @@ -46,6 +46,7 @@ from langgraph.channels.base import ( BaseChannel, ChannelsManager, EmptyChannelError, + InvalidUpdateError, create_checkpoint, ) from langgraph.channels.last_value import LastValue @@ -194,10 +195,19 @@ class Pregel( @property def config_specs(self) -> list[ConfigurableFieldSpec]: - return get_unique_config_specs( - [spec for node in self.nodes.values() for spec in node.config_specs] - + (self.checkpointer.config_specs if self.checkpointer is not None else []) - ) + return [ + spec + for spec in get_unique_config_specs( + [spec for node in self.nodes.values() for spec in node.config_specs] + + ( + self.checkpointer.config_specs + if self.checkpointer is not None + else [] + ) + ) + # these are provided by the Pregel class + if spec.id not in [CONFIG_KEY_READ, CONFIG_KEY_SEND] + ] @property def InputType(self) -> Any: @@ -684,7 +694,12 @@ def _apply_writes( # Apply writes to channels for chan, vals in pending_writes_by_channel.items(): if chan in channels: - channels[chan].update(vals) + try: + channels[chan].update(vals) + except InvalidUpdateError as e: + raise InvalidUpdateError( + f"Invalid update for channel {chan}: {e}" + ) from e checkpoint["channel_versions"][chan] += 1 updated_channels.add(chan) else: diff --git a/langgraph/pregel/read.py b/langgraph/pregel/read.py index 2baea3fa9..08903082e 100644 --- a/langgraph/pregel/read.py +++ b/langgraph/pregel/read.py @@ -16,9 +16,9 @@ from langchain_core.runnables.base import ( RunnableEach, coerce_to_runnable, ) +from langchain_core.runnables.config import merge_configs from langchain_core.runnables.utils import ConfigurableFieldSpec -from langgraph.channels.base import BaseChannel from langgraph.constants import CONFIG_KEY_READ @@ -33,7 +33,7 @@ class ChannelRead(RunnableLambda): name=CONFIG_KEY_READ, description=None, default=None, - annotation=Callable[[BaseChannel], Any], + annotation=None, ), ] @@ -103,7 +103,7 @@ class ChannelInvoke(RunnableBindingBase): when=when, bound=bound or default_bound, kwargs=kwargs or {}, - config={**(config or {}), "tags": tags or []}, + config=merge_configs(config, {"tags": tags or []}), **other_kwargs, ) diff --git a/langgraph/pregel/write.py b/langgraph/pregel/write.py index e44b3e3db..a43d8d876 100644 --- a/langgraph/pregel/write.py +++ b/langgraph/pregel/write.py @@ -43,7 +43,7 @@ class ChannelWrite(RunnablePassthrough): name=CONFIG_KEY_SEND, description=None, default=None, - annotation=TYPE_SEND, + annotation=None, ), ] diff --git a/pyproject.toml b/pyproject.toml index 97c8bc544..5dae9d362 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langgraph" -version = "0.0.12" +version = "0.0.15" description = "langgraph" authors = [] license = "LangGraph License" diff --git a/tests/test_pregel.py b/tests/test_pregel.py index fd4a986b5..1dc57988e 100644 --- a/tests/test_pregel.py +++ b/tests/test_pregel.py @@ -1,5 +1,6 @@ import operator import time +import warnings from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from typing import Annotated, Generator, Optional, TypedDict, Union @@ -43,6 +44,13 @@ def test_invoke_single_process_in_out(mocker: MockerFixture) -> None: assert app.input_schema.schema() == {"title": "LangGraphInput", "type": "integer"} assert app.output_schema.schema() == {"title": "LangGraphOutput", "type": "integer"} + with warnings.catch_warnings(): + warnings.simplefilter("error") # raise warnings as errors + assert app.config_schema().schema() == { + "properties": {}, + "title": "LangGraphConfig", + "type": "object", + } assert app.invoke(2) == 3 assert app.invoke(2, output_keys=["output"]) == {"output": 3} assert repr(app), "does not raise recursion error" @@ -811,7 +819,7 @@ def test_conditional_graph_state() -> None: ] ) - def agent_parser(input: str) -> Union[AgentAction, AgentFinish]: + def agent_parser(input: str) -> dict[str, Union[AgentAction, AgentFinish]]: if input.startswith("finish"): _, answer = input.split(":") return { diff --git a/tests/test_pregel_async.py b/tests/test_pregel_async.py index 6799bdd73..1b162cbb8 100644 --- a/tests/test_pregel_async.py +++ b/tests/test_pregel_async.py @@ -865,7 +865,7 @@ async def test_conditional_graph_state() -> None: ] ) - def agent_parser(input: str) -> Union[AgentAction, AgentFinish]: + def agent_parser(input: str) -> dict[str, Union[AgentAction, AgentFinish]]: if input.startswith("finish"): _, answer = input.split(":") return {