diff --git a/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb b/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb index fd17e3481..ff65ad927 100644 --- a/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb +++ b/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb @@ -5,13 +5,19 @@ "id": "a3e3ebc4-57af-4fe4-bdd3-36aff67bf276", "metadata": {}, "source": [ - "## Agent Evaluation\n", + "## Chat Bot Evaluation as Multi-agent Simulation\n", "\n", - "When building a chat bot, it can be hard to evaluate the multi-turn interactions.\n", + "When building a chat bot, such as a customer support assistant, it can be hard to properly evalute your bot's performance. It's time-consuming to have to manually interact with it intensively for each code change.\n", "\n", - "One way to evaluate is by simulating a conversation with a user, then evaluating the outpucomes and trajectory.\n", + "One way to make the evaluation process easier and more reproducible is to simulate a user interaction.\n", "\n", - "Below is an example of how to do this in langgraph." + "With LangGraph, it's easy to set this up. Below is an example of how to create a \"virtual user\" to simulate a conversation.\n", + "\n", + "The overall simulation looks something like this:\n", + "\n", + "![diagram](./img/virtual_user_diagram.png)\n", + "\n", + "**First,** we'll set up our environment." ] }, { @@ -21,8 +27,8 @@ "metadata": {}, "outputs": [], "source": [ - "%%capture --no-stderr\n", - "%pip install -U langchain langchain_openai langsmith pandas" + "# %%capture --no-stderr\n", + "# %pip install -U langgraph langchain langchain_openai" ] }, { @@ -44,7 +50,6 @@ "\n", "_set_if_undefined(\"OPENAI_API_KEY\")\n", "_set_if_undefined(\"LANGCHAIN_API_KEY\")\n", - "_set_if_undefined(\"TAVILY_API_KEY\")\n", "\n", "# Optional, add tracing in LangSmith.\n", "# This will help you visualize and debug the control flow\n", @@ -54,370 +59,346 @@ }, { "cell_type": "markdown", - "id": "321312b4-a1f0-4454-a481-fdac4e37cb7d", + "id": "6ef4528d-6b2a-47c7-98b5-50f14984a304", "metadata": {}, "source": [ - "## Make a user proxy \n", + "## 1. Your Agent API\n", "\n", - "Really unstructured right now. Will have to think about the UX of this further." + "For this notebook, we assume your agent's API accepts a list of messages and responds with a message.\n", + "This is configurable and can run on another system (e.g., if your system isn't running in python)." ] }, { "cell_type": "code", "execution_count": 3, - "id": "74bbe7c1-9e62-4bfe-9391-5b1a6bb4e6dc", + "id": "828479af-cf9c-4888-a365-599643a96b55", "metadata": {}, "outputs": [], "source": [ - "# Create a human proxy\n", - "import operator\n", - "from typing import Annotated, List, TypedDict\n", + "from typing import List\n", "\n", - "from langchain_core.messages import AIMessage, BaseMessage, HumanMessage\n", - "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n", - "\n", - "from langgraph.graph import END, StateGraph\n", + "import openai\n", "\n", "\n", - "def create_user_proxy(llm):\n", - " # We would likely want to provide more system prompt customization here\n", - " prompt = ChatPromptTemplate.from_messages(\n", - " [\n", - " (\n", - " \"system\",\n", - " \"You are role-playing the human character: {name}. \"\n", - " \"You are not an AI assistant and you are not supposed to help or assist.\"\n", - " \" You must behave as this human would throughout the conversation below.\"\n", - " \" Your response will be labeled as a human message.\"\n", - " \" You will be evaluated based on how realistic your impersonation of\"\n", - " \" this character is. Here are the details for your character, {name}:\\n\"\n", - " \"{system_prompt}\"\n", - " '\\n\\nWhen you are finished with the conversation, respond with a single word \"FINISHED\"',\n", - " ),\n", - " MessagesPlaceholder(variable_name=\"messages\"),\n", - " ]\n", + "# This is flexible, but you can define your agent here, or call your agent API here.\n", + "def my_chat_bot(messages: List[dict]) -> dict:\n", + " completion = openai.chat.completions.create(\n", + " messages=messages, model=\"gpt-3.5-turbo\"\n", " )\n", - " return prompt | llm | (lambda x: {\"messages\": [HumanMessage(content=x.content)]})" + " return completion.choices[0].message.model_dump()" ] }, { "cell_type": "markdown", - "id": "58480cbf-33dc-48ea-85a4-a4436e3fc283", + "id": "321312b4-a1f0-4454-a481-fdac4e37cb7d", "metadata": {}, "source": [ - "## Define the simulation constructor\n", - "\n", - "Construct the graph that connects the user proxy and the evaluated model." + "## 2. Define the Agent Simulation" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 12, "id": "cc93d114-fa65-4021-a67e-b1e6d4edc88a", "metadata": {}, "outputs": [], "source": [ - "from typing import Callable\n", + "import operator\n", + "from typing import Annotated, Callable, Dict, List, TypedDict\n", "\n", + "from langchain.adapters.openai import convert_message_to_dict\n", + "from langchain_core.messages import AIMessage, BaseMessage, HumanMessage\n", + "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n", "from langchain_core.runnables import chain\n", "from langchain_openai import ChatOpenAI\n", "\n", + "from langgraph.graph import END, StateGraph\n", "\n", + "SIMULATED_USER_NAME = \"simulated\"\n", + "\n", + "\n", + "# This is just an example, we can\n", + "# configure additional parameters if\n", + "# you want more control\n", + "class SimulatedUserConfig(TypedDict):\n", + " system_prompt: str\n", + "\n", + "\n", + "# This is the input to every node in the simulation graph\n", + "# It tracks the graph state over time. Our only \"state\"\n", + "# is the conversation messages, while the user config\n", + "# is provided to make the virtual user more unique or realistic\n", "class Environment(TypedDict):\n", " messages: Annotated[List[BaseMessage], operator.add]\n", - " system_prompt: str\n", - " name: str\n", - " input: str\n", + " simulated_user_config: SimulatedUserConfig\n", "\n", "\n", + "# We currently let the virtual user decide if the conversation can end\n", + "# we could also track max conversation turns, add a conversation \"supervisor\"\n", + "# or use other heuristics to control the dialogue flow\n", "def should_continue(state: Environment):\n", + " \"\"\"Determine if the simulation should continue.\"\"\"\n", " if state[\"messages\"][-1].content.strip().endswith(\"FINISHED\"):\n", " return \"end\"\n", " return \"continue\"\n", "\n", "\n", + "## The next two functions define the API between the simulation\n", + "# and the chat bot you wish to test.\n", + "# We are assuming your chat bot accepts a list of OAI messages\n", "@chain\n", - "def next_message(state: Environment):\n", - " return {\"input\": state[\"messages\"][-1].content}\n", + "def get_messages_for_agent(state: Environment):\n", + " \"\"\"Convert the simulation state to the input\n", + "\n", + " for your agent you want to evaluate.\"\"\"\n", + " return [convert_message_to_dict(message) for message in state[\"messages\"]]\n", "\n", "\n", + "# This takes the output of your chat bot\n", + "# and adds it to the simulation state\n", + "def get_response_message_from_agent(agent_output):\n", + " \"\"\"Get the response from the agent you are evaluting,\n", + " and use it to update the simulation state.\"\"\"\n", + " # If we do an ai message here, the user proxy llm\n", + " # will usually forget it's acting.\n", + " return {\"messages\": [HumanMessage(content=agent_output[\"content\"])]}\n", + "\n", + "\n", + "# This is run once at the beginning of the simulation.\n", + "# It's more convenient to just write an input string\n", + "# than to pass in a full message, but this could be removed below\n", "def enter(inputs: dict):\n", - " inputs[\"messages\"] = [HumanMessage(content=inputs[\"input\"], name=inputs[\"name\"])]\n", + " \"\"\"Start the simulation. This makes it less verbose to invoke.\"\"\"\n", + " inputs[\"messages\"] = [\n", + " HumanMessage(content=inputs[\"input\"], name=SIMULATED_USER_NAME)\n", + " ]\n", " return inputs\n", "\n", "\n", - "def exit(agent_output):\n", - " # If we do an ai message here, the user proxy llm\n", - " # will usually forget it's acting.\n", - " return {\"messages\": [HumanMessage(content=agent_output[\"output\"])]}\n", - "\n", - "\n", - "def create_simulation(user_proxy_llm, agent_constructor: Callable):\n", - " user_proxy = create_user_proxy(user_proxy_llm)\n", - " agent_to_evaluate = agent_constructor()\n", + "def create_simulation(chat_bot: Callable[[List[Dict]], Dict], simulated_user_llm=None):\n", + " \"\"\"Create a chat bot simulation graph.\n", "\n", + " Args:\n", + " - chat_bot: the agent you are evaluating. Accepts a list of openai messages\n", + " and returns an openai assistant message\n", + " - simulated_user_llm: the LLM to power your virtual user.\n", + " Defaults to gpt-4-1106-preview\n", + " Returns:\n", + " - simulation: an runnable object formed from compiling the state graph\n", + " \"\"\"\n", + " # This defines the virtual user proxy\n", + " prompt = ChatPromptTemplate.from_messages(\n", + " [\n", + " (\n", + " \"system\",\n", + " \"You are role-playing a human character: '{name}'. \"\n", + " \"You are not an AI assistant and you are not supposed to help or assist.\"\n", + " \" You must behave as this human would throughout the conversation below.\\n\\n\"\n", + " \"Your messages will bear the name 'simulated', but DO NOT under any circumstances\"\n", + " \"say that you are 'simulated'. You will be evaluated based on how realistic your\"\n", + " \"impersonation of this character is. This must feel real! Here are the details for your character:\"\n", + " \"\\n\"\n", + " \"{system_prompt}\" # This is the value you provide to characterize the user\n", + " '\\n\\nWhen you are finished with the conversation, respond with a single word \"FINISHED\"',\n", + " ),\n", + " MessagesPlaceholder(variable_name=\"messages\"),\n", + " ]\n", + " ).partial(name=SIMULATED_USER_NAME)\n", + " simulated_user_llm = simulated_user_llm or ChatOpenAI(model=\"gpt-4-1106-preview\")\n", + " user_proxy = (\n", + " (lambda x: {**x, **x[\"simulated_user_config\"]})\n", + " | prompt\n", + " | simulated_user_llm\n", + " | (\n", + " lambda x: {\n", + " \"messages\": [HumanMessage(content=x.content, name=SIMULATED_USER_NAME)]\n", + " }\n", + " )\n", + " )\n", " graph_builder = StateGraph(Environment)\n", " graph_builder.add_node(\"user\", user_proxy)\n", - " # give it a single string input\n", - " graph_builder.add_node(\"agent\", next_message | agent_to_evaluate | exit)\n", - " graph_builder.add_edge(\"agent\", \"user\")\n", + " graph_builder.add_node(\n", + " # The \"|\" syntax composes these steps in the pipeline to map between\n", + " # the simulation state and your chat bot's API\n", + " \"chat_bot\",\n", + " get_messages_for_agent | chat_bot | get_response_message_from_agent,\n", + " )\n", + " # Every response from your chat bot will automatically go to the\n", + " # simulated user\n", + " graph_builder.add_edge(\"chat_bot\", \"user\")\n", " graph_builder.add_conditional_edges(\n", " \"user\",\n", " should_continue,\n", + " # If the finish criteria are met, we will stop the simulation,\n", + " # otherwise, the virtual user's message will be sent to your chat bot\n", " {\n", " \"end\": END,\n", - " \"continue\": \"agent\",\n", + " \"continue\": \"chat_bot\",\n", " },\n", " )\n", - " graph_builder.set_entry_point(\"agent\")\n", + " # The input will first go to your chat bot\n", + " graph_builder.set_entry_point(\"chat_bot\")\n", " return (enter | graph_builder.compile()).with_config(run_name=\"Agent Simulation\")" ] }, { "cell_type": "markdown", - "id": "243d07dd-187c-4cc7-815d-dee4fbe84df4", + "id": "2e0bd26e-8c1d-471d-9fef-d95dc0163491", "metadata": {}, "source": [ - "## Create your agent\n", + "## Run Simulation\n", "\n", - "We make a constructor. There's some additional data munging to make the simulator output work with your model." + "Now we can evaluate our chat bot!" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "c462eac5-53b4-4ea3-b38a-40940826df6b", + "execution_count": 13, + "id": "4d495ccc-0f5f-4194-89e5-15dccbbb7412", "metadata": {}, "outputs": [], "source": [ - "from langchain_core.output_parsers import StrOutputParser\n", - "from langchain_core.runnables.history import RunnableWithMessageHistory\n", - "\n", - "\n", - "# This returns any runnable. It would probably accept \"input\" or something\n", - "# It's the agent you want to evaluate\n", - "def creat_conversational_agent():\n", - " messages = []\n", - "\n", - " def add_user_message(inputs):\n", - " messages.append(HumanMessage(content=inputs[\"input\"]))\n", - " return {\"messages\": messages}\n", - "\n", - " def add_ai_message(output):\n", - " messages.append(output)\n", - " return {\"output\": output.content}\n", - "\n", - " chain = (\n", - " add_user_message\n", - " | ChatPromptTemplate.from_messages(\n", - " [\n", - " (\"system\", \"you are a helpful assistant\"),\n", - " MessagesPlaceholder(variable_name=\"messages\"),\n", - " ]\n", - " )\n", - " | ChatOpenAI(model=\"gpt-3.5-turbo\")\n", - " | add_ai_message\n", - " )\n", - " return chain" - ] - }, - { - "cell_type": "markdown", - "id": "219859c4-3785-4680-8738-9512a64be3bf", - "metadata": {}, - "source": [ - "## Create a dataset" + "simulation = create_simulation(my_chat_bot)" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "394c19df-05fa-47df-8e18-a86c0bc4e138", - "metadata": {}, - "outputs": [], - "source": [ - "import langsmith\n", - "\n", - "client = langsmith.Client()\n", - "\n", - "examples = [\n", - " (\n", - " \"Adam\",\n", - " \"You really needs to find the restroom. You only speak English.\",\n", - " \"Hi there, where's the restroom?\",\n", - " ),\n", - " (\n", - " \"Steve\",\n", - " \"You are a developer who needs help with an error.\",\n", - " \"How do I fix this? NameError: module terminator cannot be found\",\n", - " ),\n", - " (\"Jezebel\", \"You want to learn more about langchain\", \"So what's langchain?\"),\n", - "]\n", - "\n", - "dataset_name = \"Agent Simulation\"\n", - "if not client.has_dataset(dataset_name=dataset_name):\n", - " ds = client.create_dataset(dataset_name=dataset_name)\n", - " client.create_examples(\n", - " inputs=[\n", - " {\"system_prompt\": sys, \"name\": name, \"input\": input}\n", - " for name, sys, input in examples\n", - " ],\n", - " dataset_id=ds.id,\n", - " )\n", - "# else:\n", - "# client.delete_dataset(dataset_name=dataset_name)" - ] - }, - { - "cell_type": "markdown", - "id": "990f198d-29c9-400b-b7e8-c6c5a0b7ef77", - "metadata": {}, - "source": [ - "## Define evaluators\n", - "\n", - "We will use an LLM as judge here. In reality, you would probably add additional\n", - "context to the ground truth to make the results more reliable." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "d8b1ebd8-60c5-4051-85d8-2576c81b3b41", - "metadata": {}, - "outputs": [], - "source": [ - "from langchain.load import load\n", - "from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser\n", - "from langchain_core.pydantic_v1 import BaseModel, Field\n", - "from langsmith.evaluation import run_evaluator\n", - "\n", - "\n", - "class GradeConversation(BaseModel):\n", - " \"\"\"Grade the conversation based on quality.\"\"\"\n", - "\n", - " reasoning: Annotated[\n", - " str, \"The step-by-step reasoning explaining the conversation quality score\"\n", - " ]\n", - " score: Annotated[int, \"The score on a scale from 0 to 5\"] = Field(gte=0, lte=5)\n", - "\n", - "\n", - "@run_evaluator\n", - "def conversation_quality(run, example):\n", - " if not run.outputs:\n", - " return {\"score\": 0, \"comment\": \"Unfinished\"}\n", - " # TODO: Incorporate examples\n", - " prompt = ChatPromptTemplate.from_messages(\n", - " [\n", - " (\n", - " \"system\",\n", - " \"You are grading the quality of the following conversation.\"\n", - " \"Submit your response using the GradeConversation function.\",\n", - " ),\n", - " MessagesPlaceholder(variable_name=\"messages\"),\n", - " (\"system\", \"Review the conversation above and submit your score.\"),\n", - " ]\n", - " )\n", - " bound = ChatOpenAI(model=\"gpt-4-1106-preview\").bind_functions(\n", - " functions=[GradeConversation], function_call=\"GradeConversation\"\n", - " )\n", - " chain = prompt | bound | JsonOutputFunctionsParser()\n", - " result = chain.invoke(load(run.outputs))\n", - " score = result[\"score\"] / 5\n", - " return {\"score\": score, \"comment\": result[\"reasoning\"]}" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "39d6287f-2768-42f5-a428-d753ece6048a", - "metadata": {}, - "outputs": [], - "source": [ - "# llm = ChatOpenAI(model=\"gpt-4-1106-preview\")\n", - "# graph = create_simulation(llm, creat_conversational_agent)\n", - "\n", - "# graph.invoke(\n", - "# {\n", - "# \"system_prompt\": \"You really needs to find the restroom. You only speak English.\",\n", - "# \"name\": \"Adam\",\n", - "# \"input\": \"I really need to go\",\n", - "# }\n", - "# )" - ] - }, - { - "cell_type": "markdown", - "id": "ed54f4c9-e52d-421f-935d-02aab5d5d6cb", - "metadata": {}, - "source": [ - "## Run Evaluation" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "be206282-2d7c-4f5c-9c71-77dd9e192b6d", + "execution_count": 14, + "id": "fa91e733-3493-43c2-ba21-171cf78deef8", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/Users/wfh/.pyenv/versions/3.11.2/lib/python3.11/site-packages/langchain/smith/evaluation/runner_utils.py:1244: LangChainDeprecationWarning: The following arguments are deprecated and will be removed in a future release: dict_keys(['revision_id']).\n", - " warn_deprecated(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "View the evaluation results for project 'ample-color-99' at:\n", - "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/e5fde795-82eb-4ca2-8f2e-9afa666b5c0f/compare?selectedSessions=61f4e55a-1496-49e7-8ef3-e39376a64d8a\n", - "\n", - "View all tests for Dataset Agent Simulation at:\n", - "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/e5fde795-82eb-4ca2-8f2e-9afa666b5c0f\n", - "[> ] 0/3" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/wfh/code/lc/langchain/libs/core/langchain_core/_api/beta_decorator.py:86: LangChainBetaWarning: The function `load` is in beta. It is actively being worked on, so the API may change.\n", - " warn_beta(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[------------------------------------------------->] 3/3" + "Skipping write for channel input which has no readers\n" ] } ], "source": [ - "import functools\n", + "from langchain_core.tracers.context import tracing_v2_enabled\n", "\n", - "from langchain.smith import RunEvalConfig\n", + "# The tracing context manager lets us easily fetch the trace URL in-context.\n", + "# You can turn this off if you don't want to trace the execution.\n", + "with tracing_v2_enabled() as tracer:\n", + " result = simulation.invoke(\n", + " {\n", + " \"simulated_user_config\": {\n", + " \"system_prompt\": \"You are on a budget. Your family is hard to please.\"\n", + " \" They all like the beach, except for Aunt Lily, who prefers the mountains.\"\n", + " },\n", + " \"input\": \"help me plan my family vacation\",\n", + " }\n", + " )\n", + " # You can go to this run to review the entire simulation trace\n", + " url = tracer.get_run_url()" + ] + }, + { + "cell_type": "markdown", + "id": "73ff30e4-1992-4bc9-834d-f4c08b281d20", + "metadata": {}, + "source": [ + "### Review Result\n", "\n", - "config = RunEvalConfig(\n", - " custom_evaluators=[conversation_quality],\n", - ")\n", - "results = client.run_on_dataset(\n", - " dataset_name=dataset_name,\n", - " llm_or_chain_factory=functools.partial(\n", - " create_simulation,\n", - " ChatOpenAI(model=\"gpt-4-1106-preview\"),\n", - " creat_conversational_agent,\n", - " ),\n", - " evaluation=config,\n", + "If you've traced the run, you can see the resulting trace in the UI by clicking on the url.\n", + "Select the last 'ChatOpenAI' call in the trace to see the full conversation in a single view.\n", + "\n", + "![full-conversation](./img/virtual_user_full_convo.png)\n", + "\n", + "\n", + "From this run, you can manually annotate it to score its quality.\n", + "\n", + "![annotate](./img/virtual_user_annotate.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "702d008c-dac5-478f-8239-ede3050573c6", + "metadata": {}, + "outputs": [], + "source": [ + "url" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "c1037dd1-7e2e-4158-b248-bc7a8839e62e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/191a0aa8-051e-41e6-91f8-1a213087e358/r/d3f6e590-bae8-402e-83a7-46739279c061?poll=true'" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "url" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "0de1664f-1625-42e7-9dad-eb2c061b88d4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[HumanMessage(content='help me plan my family vacation', name='simulated'),\n", + " HumanMessage(content=\"Of course! I'd be happy to help you plan your family vacation. Please provide me with some additional information about your preferences, such as your desired destination, budget, travel dates, and any specific activities or interests you would like to incorporate into your trip.\"),\n", + " HumanMessage(content='simulated', name='simulated'),\n", + " HumanMessage(content=\"Sure! Even though I cannot physically make bookings or arrangements, I can certainly help you with the planning process and give you some suggestions. Please let me know the following details:\\n\\n1. Destination: Where would you like to go? Do you have a specific country, city, or region in mind?\\n\\n2. Duration: How long are you planning to go on vacation? Do you have specific dates in mind?\\n\\n3. Budget: Have you determined a budget for your trip? Knowing your budget will help me provide recommendations within your means.\\n\\n4. Interests: What are some of the activities or attractions that your family enjoys? Are there any specific interests or hobbies you'd like to incorporate into your vacation?\\n\\n5. Accommodation preferences: What type of accommodation are you looking for? Are you open to hotels, vacation rentals, or maybe even camping?\\n\\n6. Mode of transportation: How do you plan on getting to your destination and moving around? Will you be driving, flying, or using public transportation?\\n\\n7. Number of travelers: How many people are in your family and are there any specific needs or requirements we should consider?\\n\\nOnce you provide me with this information, I'll be able to give you some personalized suggestions and tips to help you plan your family vacation.\"),\n", + " HumanMessage(content=\"It's always a bit of a juggling act to plan a family vacation with different preferences, but we can definitely work something out. Since everyone likes the beach except Aunt Lily, who prefers the mountains, we might need to find a compromise. Maybe we could aim for a coastal area that also has access to mountainous or hilly terrain. \\n\\nThis way, we could spend some days at the beach and other days exploring the mountains. For instance, somewhere like the Central Coast of California might be a good fit. You've got beaches, but you're also not too far from areas like Big Sur, which offers stunning mountain views and hikes.\\n\\nGiven the budget constraints, we'd probably want to look for affordable accommodation. A vacation rental might give us more for our money than a hotel, especially if we have a kitchen to prepare some of our own meals. \\n\\nAs for activities, we could mix it up with some free or low-cost options like hiking, beach days, and exploring local towns. Maybe even find a beach with some nearby trails or scenic drives that give you a bit of that mountain feel. \\n\\nDoes this sound like something that might work for your family? What's your budget looking like, and how many people are we planning for? That way, I can keep my suggestions realistic.\", name='simulated'),\n", + " HumanMessage(content=\"That sounds like a great plan! The Central Coast of California is a beautiful destination that offers a mix of beach and mountain activities. It's an excellent compromise for your family's preferences.\\n\\nIn terms of budget, it's helpful to have a rough estimate of how much you are willing to spend on accommodation, transportation, activities, and meals. This will allow me to provide you with suitable options within your budget.\\n\\nAdditionally, please let me know the number of people in your family and any specific requirements or preferences that I should take into account when making suggestions, such as the need for wheelchair accessibility or child-friendly activities.\\n\\nOnce I have these details, I can provide you with specific recommendations for places to stay, activities to do, and sights to see along the Central Coast of California.\"),\n", + " HumanMessage(content='simulated', name='simulated'),\n", + " HumanMessage(content=\"Assuming you're aiming for a budget-friendly family vacation, here's a suggested itinerary for your trip to the Central Coast of California:\\n\\n1. Destination: Pismo Beach\\n - Pismo State Beach: Enjoy a day of relaxation on the beautiful sandy beaches of Pismo State Beach, perfect for swimming, sunbathing, and picnicking.\\n - Monarch Butterfly Grove: Visit the Monarch Butterfly Grove during the winter months to witness thousands of butterflies in their natural habitat.\\n\\n2. Destination: Morro Bay\\n - Morro Bay State Park: Explore the bay and its surrounding trails for hiking and wildlife spotting.\\n - Morro Bay Harborwalk: Take a leisurely stroll along the harbor, visit the charming shops, and enjoy fresh seafood at one of the waterfront restaurants.\\n\\n3. Destination: San Luis Obispo\\n - Bubblegum Alley: Marvel at this unique attraction where the walls are covered in chewed-up bubblegum.\\n - San Luis Obispo Children's Museum: If you have kids, they'll love this interactive and educational museum.\\n\\n4. Destination: Cambria\\n - Moonstone Beach: Search for moonstones and take a walk along the beautiful coastal boardwalk.\\n - Hearst Castle: Visit the historic Hearst Castle, a grand mansion with stunning architecture and beautiful gardens.\\n\\n5. Destination: Big Sur\\n - Julia Pfeiffer Burns State Park: Hike the iconic Overlook Trail and witness the breathtaking McWay Falls, a majestic waterfall that cascades onto the beach.\\n - Pfeiffer Beach: Spend a peaceful afternoon at Pfeiffer Beach, known for its purple sand and rock formations.\\n\\nAs for accommodation, consider looking for vacation rentals or affordable hotels in each destination. Booking.com or Airbnb can be useful resources for finding budget-friendly options.\\n\\nRemember to check for any travel advisories or restrictions, and to plan activities based on the interests and abilities of your family members.\\n\\nI hope this suggested itinerary helps you plan your family vacation to the Central Coast of California! Let me know if you need any more information or assistance. Safe travels!\"),\n", + " HumanMessage(content='FINISHED', name='simulated')]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result[\"messages\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e871cb1f-b3d4-44d7-83a8-c61f65898568", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a046369d-bce9-4215-9d1a-e5522d586916", + "metadata": {}, + "outputs": [], + "source": [ + "result = simulation.invoke(\n", + " {\n", + " \"simulated_user_config\": {\"system_prompt\": \"You are \"},\n", + " \"input\": \"help me plan my family vacation\",\n", + " }\n", ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "56a40a17-abba-4f12-800a-945a81bc6fa1", + "id": "dde4f2b5-cfe8-4ff0-99ea-fe2c5fed70c0", "metadata": {}, "outputs": [], "source": [] diff --git a/examples/advanced_agents/multi-agent/img/virtual_user_annotate.png b/examples/advanced_agents/multi-agent/img/virtual_user_annotate.png new file mode 100644 index 000000000..8886efae4 Binary files /dev/null and b/examples/advanced_agents/multi-agent/img/virtual_user_annotate.png differ diff --git a/examples/advanced_agents/multi-agent/img/virtual_user_diagram.png b/examples/advanced_agents/multi-agent/img/virtual_user_diagram.png new file mode 100644 index 000000000..a0cefb06b Binary files /dev/null and b/examples/advanced_agents/multi-agent/img/virtual_user_diagram.png differ diff --git a/examples/advanced_agents/multi-agent/img/virtual_user_full_convo.png b/examples/advanced_agents/multi-agent/img/virtual_user_full_convo.png new file mode 100644 index 000000000..8da9c2d3c Binary files /dev/null and b/examples/advanced_agents/multi-agent/img/virtual_user_full_convo.png differ