WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-08-26 01:22:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

agent simulation clean up

Browse Source
This commit is contained in:
Harrison Chase
2024-01-21 09:03:39 -08:00
parent 6236fb086d
commit f71943952e
2 changed files with 259 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb
+257 -241
View File
@@ -84,172 +84,265 @@
"\n",
"# This is flexible, but you can define your agent here, or call your agent API here.\n",
"def my_chat_bot(messages: List[dict]) -> dict:\n",
" system_message = {\"role\": \"system\", \"content\": \"You are a customer support agent for an airline.\"}\n",
" messages = [system_message] + messages\n",
" completion = openai.chat.completions.create(\n",
" messages=messages, model=\"gpt-3.5-turbo\"\n",
" )\n",
" return completion.choices[0].message.model_dump()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "f58959bf-2ab5-4330-9ac2-c00f45237e24",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'content': 'Hello! How can I assist you today?',\n",
" 'role': 'assistant',\n",
" 'function_call': None,\n",
" 'tool_calls': None}"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"my_chat_bot([{\"role\": \"user\", \"content\": \"hi!\"}])"
]
},
{
"cell_type": "markdown",
"id": "321312b4-a1f0-4454-a481-fdac4e37cb7d",
"id": "419340a3-5ecf-48e7-9028-4f2fad750502",
"metadata": {},
"source": [
"## 2. Define the Agent Simulation\n",
"## 2. Define Simulated User\n",
"\n",
"The code below creates a LangGraph workflow to run the simulation. The main components are:\n",
"\n",
"1. Simulation state: the inputs to each node in the graph, containing the messages (stateful) and the details about the simulated user.\n",
"2. Functions to interoperate between the simulation state and your chat bot's API\n",
"3. The simulated user definition.\n",
"4. The graph itself, with a conditional stopping criterion.\n",
"\n",
"Read the comments in the code below for more information."
"We're now going to define the simulated user. \n",
"This can be anything we want, but we're going to build it as a LangChain bot."
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "cc93d114-fa65-4021-a67e-b1e6d4edc88a",
"execution_count": 5,
"id": "32c147df-7f90-4b0d-9a6b-671677020353",
"metadata": {},
"outputs": [],
"source": [
"import operator\n",
"from typing import Annotated, Callable, Dict, List, TypedDict\n",
"\n",
"from langchain.adapters.openai import convert_message_to_dict\n",
"from langchain_core.messages import AIMessage, BaseMessage, HumanMessage\n",
"from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
"from langchain_core.runnables import chain\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"from langgraph.graph import END, StateGraph\n",
"system_prompt_template = \"\"\"You are a customer of an airline company. \\\n",
"You are interacting with a user who is a customer support person. \\\n",
"\n",
"SIMULATED_USER_NAME = \"simulated\"\n",
"{instructions}\n",
"\n",
"When you are finished with the conversation, respond with a single word 'FINISHED'\"\"\"\n",
"\n",
"# This is just an example, we can\n",
"# configure additional parameters if\n",
"# you want more control\n",
"class SimulatedUserConfig(TypedDict):\n",
" system_prompt: str\n",
"\n",
"\n",
"# This is the input to every node in the simulation graph\n",
"# It tracks the graph state over time. Our only \"state\"\n",
"# is the conversation messages, while the user config\n",
"# is provided to make the virtual user more unique or realistic\n",
"class Environment(TypedDict):\n",
" messages: Annotated[List[BaseMessage], operator.add]\n",
" simulated_user_config: SimulatedUserConfig\n",
"\n",
"\n",
"# We currently let the virtual user decide if the conversation can end\n",
"# we could also track max conversation turns, add a conversation \"supervisor\"\n",
"# or use other heuristics to control the dialogue flow\n",
"def should_continue(state: Environment):\n",
" \"\"\"Determine if the simulation should continue.\"\"\"\n",
" if state[\"messages\"][-1].content.strip().endswith(\"FINISHED\"):\n",
" return \"end\"\n",
" return \"continue\"\n",
"\n",
"\n",
"## The next two functions define the API between the simulation\n",
"# and the chat bot you wish to test.\n",
"# We are assuming your chat bot accepts a list of OAI messages\n",
"@chain\n",
"def get_messages_for_agent(state: Environment):\n",
" \"\"\"Convert the simulation state to the input\n",
"\n",
" for your agent you want to evaluate.\"\"\"\n",
" return [convert_message_to_dict(message) for message in state[\"messages\"]]\n",
"\n",
"\n",
"# This takes the output of your chat bot\n",
"# and adds it to the simulation state\n",
"def get_response_message_from_agent(agent_output):\n",
" \"\"\"Get the response from the agent you are evaluting,\n",
" and use it to update the simulation state.\"\"\"\n",
" # If we do an ai message here, the user proxy llm\n",
" # will usually forget it's acting.\n",
" return {\"messages\": [HumanMessage(content=agent_output[\"content\"])]}\n",
"\n",
"\n",
"# This is run once at the beginning of the simulation.\n",
"# It's more convenient to just write an input string\n",
"# than to pass in a full message, but this could be removed below\n",
"def enter(inputs: dict):\n",
" \"\"\"Start the simulation. This makes it less verbose to invoke.\"\"\"\n",
" inputs[\"messages\"] = [\n",
" HumanMessage(content=inputs[\"input\"], name=SIMULATED_USER_NAME)\n",
"prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\"system\", system_prompt_template),\n",
" MessagesPlaceholder(variable_name=\"messages\"),\n",
" ]\n",
" return inputs\n",
")\n",
"instructions = \"\"\"Your name is Harrison. You are tyring to get a refund for the trip you took to Alaska. \\\n",
"You want them to give you ALL the money back. \\\n",
"This trip happened 5 years ago.\"\"\"\n",
"\n",
"prompt = prompt.partial(name=\"Harrison\", instructions=instructions)\n",
"\n",
"model = ChatOpenAI()\n",
"\n",
"simulated_user = prompt | model"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "6f80669e-aa78-4666-b67c-a539366d5aab",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"AIMessage(content='Hi, I would like to request a refund for a trip I took with your airline company to Alaska. Is it possible to get a refund for that trip?')"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from langchain_core.messages import HumanMessage\n",
"\n",
"\n",
"def create_simulation(chat_bot: Callable[[List[Dict]], Dict], simulated_user_llm=None):\n",
" \"\"\"Create a chat bot simulation graph.\n",
"messages = [HumanMessage(content=\"Hi! How can I help you?\")]\n",
"simulated_user.invoke({\"messages\": messages})"
]
},
{
"cell_type": "markdown",
"id": "321312b4-a1f0-4454-a481-fdac4e37cb7d",
"metadata": {},
"source": [
"## 3. Define the Agent Simulation\n",
"\n",
" Args:\n",
" - chat_bot: the agent you are evaluating. Accepts a list of openai messages\n",
" and returns an openai assistant message\n",
" - simulated_user_llm: the LLM to power your virtual user.\n",
" Defaults to gpt-4-1106-preview\n",
" Returns:\n",
" - simulation: an runnable object formed from compiling the state graph\n",
" \"\"\"\n",
" # This defines the virtual user proxy\n",
" prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\n",
" \"system\",\n",
" \"You are role-playing a human character: '{name}'. \"\n",
" \"You are not an AI assistant and you are not supposed to help or assist.\"\n",
" \" You must behave as this human would throughout the conversation below.\\n\\n\"\n",
" \"Your messages will bear the name 'simulated', but DO NOT under any circumstances\"\n",
" \"say that you are 'simulated'. You will be evaluated based on how realistic your\"\n",
" \"impersonation of this character is. This must feel real! Here are the details for your character:\"\n",
" \"\\n\"\n",
" \"{system_prompt}\" # This is the value you provide to characterize the user\n",
" '\\n\\nWhen you are finished with the conversation, respond with a single word \"FINISHED\"',\n",
" ),\n",
" MessagesPlaceholder(variable_name=\"messages\"),\n",
" ]\n",
" ).partial(name=SIMULATED_USER_NAME)\n",
" simulated_user_llm = simulated_user_llm or ChatOpenAI(model=\"gpt-4-1106-preview\")\n",
" user_proxy = (\n",
" (lambda x: {**x, **x[\"simulated_user_config\"]})\n",
" | prompt\n",
" | simulated_user_llm\n",
" | (\n",
" lambda x: {\n",
" \"messages\": [HumanMessage(content=x.content, name=SIMULATED_USER_NAME)]\n",
" }\n",
" )\n",
" )\n",
" graph_builder = StateGraph(Environment)\n",
" graph_builder.add_node(\"user\", user_proxy)\n",
" graph_builder.add_node(\n",
" # The \"|\" syntax composes these steps in the pipeline to map between\n",
" # the simulation state and your chat bot's API\n",
" \"chat_bot\",\n",
" get_messages_for_agent | chat_bot | get_response_message_from_agent,\n",
" )\n",
" # Every response from your chat bot will automatically go to the\n",
" # simulated user\n",
" graph_builder.add_edge(\"chat_bot\", \"user\")\n",
" graph_builder.add_conditional_edges(\n",
" \"user\",\n",
" should_continue,\n",
" # If the finish criteria are met, we will stop the simulation,\n",
" # otherwise, the virtual user's message will be sent to your chat bot\n",
" {\n",
" \"end\": END,\n",
" \"continue\": \"chat_bot\",\n",
" },\n",
" )\n",
" # The input will first go to your chat bot\n",
" graph_builder.set_entry_point(\"chat_bot\")\n",
" return (enter | graph_builder.compile()).with_config(run_name=\"Agent Simulation\")"
"The code below creates a LangGraph workflow to run the simulation. The main components are:\n",
"\n",
"1. Simulation state: the inputs to each node in the graph, containing the messages.\n",
"2. The two nodes: one for the simulated user, the other for the chat bot.\n",
"3. The graph itself, with a conditional stopping criterion.\n",
"\n",
"Read the comments in the code below for more information.\n"
]
},
{
"cell_type": "markdown",
"id": "65bc4446-462b-4ee8-b017-2862fbbdfaf5",
"metadata": {},
"source": [
"**Nodes**\n",
"\n",
"First, we define the nodes in the graph. These should take in a list of messages and return a list of messages to ADD to the state.\n",
"These will be thing wrappers around the chat bot and simulated user we have above.\n",
"\n",
"**Note:** one tricky thing here is which messages are which. Because both the chat bot AND our simulated user are both LLMs, both of them will resond with AI messages. Our state will be a list of alternating Human and AI messages. This means that for one of the nodes, there will need to be some logic that flips the AI and human roles. In this example, we will assume that HumanMessages are messages from the simulated user. This means that we need some logic in the simulated user node to swap AI and Human messages.\n",
"\n",
"First, let's define the chat bot node"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "69e2a3a3-40f3-4223-9136-113738440be9",
"metadata": {},
"outputs": [],
"source": [
"from langchain.adapters.openai import convert_message_to_dict\n",
"from langchain_core.messages import AIMessage\n",
"\n",
"\n",
"def chat_bot_node(messages):\n",
" # Convert from LangChain format to the OpenAI format, which our chatbot function expects.\n",
" messages = [convert_message_to_dict(m) for m in messages]\n",
" # Call the chat bot\n",
" chat_bot_response = my_chat_bot(messages)\n",
" # Respond with an AI Message\n",
" return AIMessage(content=chat_bot_response[\"content\"])\n",
" "
]
},
{
"cell_type": "markdown",
"id": "694c3c0c-56c5-4410-8fa8-ea2c0f11f506",
"metadata": {},
"source": [
"Next, let's define the node for our simulated user. This will involve a little logic to swap the roles of the messages."
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "7cad7527-ffa5-4c30-8585-b54a7a18bd98",
"metadata": {},
"outputs": [],
"source": [
"def _swap_roles(messages):\n",
" new_messages = []\n",
" for m in messages:\n",
" if isinstance(m, AIMessage):\n",
" new_messages.append(HumanMessage(content=m.content))\n",
" else:\n",
" new_messages.append(AIMessage(content=m.content))\n",
" return new_messages\n",
"\n",
"\n",
"def simulated_user_node(messages):\n",
" # Swap roles of messages\n",
" new_messages = _swap_roles(messages)\n",
" # Call the simulated user\n",
" response = simulated_user.invoke({\"messages\": new_messages})\n",
" # This response is an AI message - we need to flip this to be a human message\n",
" return HumanMessage(content=response.content)"
]
},
{
"cell_type": "markdown",
"id": "a48d8a3e-9171-4c43-a595-44d312722148",
"metadata": {},
"source": [
"**Edges**\n",
"\n",
"We now need to define the logic for the edges. The main logic occurs after the simulated user goes, and it should lead to one of two outcomes:\n",
"\n",
"- Either we continue and call the customer support bot\n",
"- Or we finish and the conversation is over\n",
"\n",
"So what is the logic for the conversation being over? We will define that as either the Human chatbot responds with `FINISHED` (see the system prompt) OR the conversation is more than 6 messages long (this is an arbitrary number just to keep this example short)."
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "28004fbf-a2f3-46b7-bde7-46c7adaf97fb",
"metadata": {},
"outputs": [],
"source": [
"def should_continue(messages):\n",
" if len(messages) > 6:\n",
" return \"end\"\n",
" elif messages[-1].content == \"FINISHED\":\n",
" return \"end\"\n",
" else:\n",
" return \"continue\""
]
},
{
"cell_type": "markdown",
"id": "d0856d4f-9334-4f28-944b-06d303e913a4",
"metadata": {},
"source": [
"**Graph**\n",
"\n",
"We can now define the graph that sets up the simulation!"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "0b597e4b-4cbb-4bbc-82e5-f7e31275964c",
"metadata": {},
"outputs": [],
"source": [
"from langgraph.graph import END, MessageGraph\n",
"\n",
"\n",
"graph_builder = MessageGraph()\n",
"graph_builder.add_node(\"user\", simulated_user_node)\n",
"graph_builder.add_node(\"chat_bot\", chat_bot_node)\n",
"# Every response from your chat bot will automatically go to the\n",
"# simulated user\n",
"graph_builder.add_edge(\"chat_bot\", \"user\")\n",
"graph_builder.add_conditional_edges(\n",
" \"user\",\n",
" should_continue,\n",
" # If the finish criteria are met, we will stop the simulation,\n",
" # otherwise, the virtual user's message will be sent to your chat bot\n",
" {\n",
" \"end\": END,\n",
" \"continue\": \"chat_bot\",\n",
" },\n",
")\n",
"# The input will first go to your chat bot\n",
"graph_builder.set_entry_point(\"chat_bot\")\n",
"simulation = graph_builder.compile()"
]
},
{
@@ -257,123 +350,46 @@
"id": "2e0bd26e-8c1d-471d-9fef-d95dc0163491",
"metadata": {},
"source": [
"## 3. Run Simulation\n",
"## 4. Run Simulation\n",
"\n",
"Now we can evaluate our chat bot! We will provide information about the simulated user (as a system prompt)\n",
"as well as the initial input message from that simulated user to the chat bot."
"Now we can evaluate our chat bot! We can invoke it with empty messages (this will simulate letting the chat bot start the initial conversation)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "4d495ccc-0f5f-4194-89e5-15dccbbb7412",
"metadata": {},
"outputs": [],
"source": [
"simulation = create_simulation(my_chat_bot)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "fa91e733-3493-43c2-ba21-171cf78deef8",
"execution_count": 11,
"id": "32848c2e-be82-46f3-81db-b23fea45461c",
"metadata": {},
"outputs": [
{
"name": "stderr",
"name": "stdout",
"output_type": "stream",
"text": [
"Skipping write for channel input which has no readers\n"
"{'chat_bot': AIMessage(content='How may I assist you today regarding your flight or any other concerns?')}\n",
"----\n",
"{'user': HumanMessage(content='Hi, my name is Harrison. I am reaching out to request a refund for a trip I took to Alaska with your airline company. The trip occurred about 5 years ago. I would like to receive a refund for the entire amount I paid for the trip. Can you please assist me with this?')}\n",
"----\n",
"{'chat_bot': AIMessage(content=\"Hello, Harrison. Thank you for reaching out to us. I understand you would like to request a refund for a trip you took to Alaska five years ago. I'm afraid that our refund policy typically has a specific timeframe within which refund requests must be made. Generally, refund requests need to be submitted within 24 to 48 hours after the booking is made, or in certain cases, within a specified cancellation period.\\n\\nHowever, I will do my best to assist you. Could you please provide me with some additional information? Can you recall any specific details about the booking, such as the flight dates, booking reference or confirmation number? This will help me further look into the possibility of processing a refund for you.\")}\n",
"----\n",
"{'user': HumanMessage(content=\"Hello, thank you for your response. I apologize for not requesting the refund earlier. Unfortunately, I don't have the specific details such as the flight dates, booking reference, or confirmation number at the moment. Is there any other way we can proceed with the refund request without these specific details? I would greatly appreciate your assistance in finding a solution.\")}\n",
"----\n",
"{'chat_bot': AIMessage(content=\"I understand the situation, Harrison. Without specific details like flight dates, booking reference, or confirmation number, it becomes challenging to locate and process the refund accurately. However, I can still try to help you.\\n\\nTo proceed further, could you please provide me with any additional information you might remember? This could include the approximate date of travel, the departure and arrival airports, the names of the passengers, or any other relevant details related to the booking. The more information you can provide, the better we can investigate the possibility of processing a refund for you.\\n\\nAdditionally, do you happen to have any documentation related to your trip, such as receipts, boarding passes, or emails from our airline? These documents could assist in verifying your trip and processing the refund request.\\n\\nI apologize for any inconvenience caused, and I'll do my best to assist you further based on the information you can provide.\")}\n",
"----\n",
"{'user': HumanMessage(content=\"I apologize for the inconvenience caused. Unfortunately, I don't have any additional information or documentation related to the trip. It seems that I am unable to provide you with the necessary details to process the refund request. I understand that this may limit your ability to assist me further, but I appreciate your efforts in trying to help. Thank you for your time. \\n\\nFINISHED\")}\n",
"----\n",
"{'chat_bot': AIMessage(content=\"I understand, Harrison. I apologize for any inconvenience caused, and I appreciate your understanding. If you happen to locate any additional information or documentation in the future, please don't hesitate to reach out to us again. Our team will be more than happy to assist you with your refund request or any other travel-related inquiries. Thank you for contacting us, and have a great day!\")}\n",
"----\n",
"{'user': HumanMessage(content='FINISHED')}\n",
"----\n"
]
}
],
"source": [
"from langchain_core.tracers.context import tracing_v2_enabled\n",
"\n",
"# The tracing context manager lets us easily fetch the trace URL in-context.\n",
"# You can turn this off if you don't want to trace the execution.\n",
"with tracing_v2_enabled() as tracer:\n",
" result = simulation.invoke(\n",
" {\n",
" \"simulated_user_config\": {\n",
" \"system_prompt\": \"You are on a budget. Your family is hard to please.\"\n",
" \" They all like the beach, except for Aunt Lily, who prefers the mountains.\"\n",
" },\n",
" \"input\": \"help me plan my family vacation\",\n",
" }\n",
" )\n",
" # You can go to this run to review the entire simulation trace\n",
" url = tracer.get_run_url()"
]
},
{
"cell_type": "markdown",
"id": "73ff30e4-1992-4bc9-834d-f4c08b281d20",
"metadata": {},
"source": [
"## (Optional) Review Results\n",
"\n",
"If you've traced the run, you can see the full simulation trace in the UI by clicking on the url.\n",
"Select the last 'ChatOpenAI' call in the trace to see the full conversation in a single view.\n",
"\n",
"![full-conversation](./img/virtual_user_full_convo.png)\n",
"\n",
"\n",
"From this run, you can manually annotate it to score its quality. This feedback can be used to compare the quality of different versions of your chat bot.\n",
"\n",
"![annotate](./img/virtual_user_annotate.png)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "702d008c-dac5-478f-8239-ede3050573c6",
"metadata": {},
"outputs": [],
"source": [
"url"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "0de1664f-1625-42e7-9dad-eb2c061b88d4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[HumanMessage(content='help me plan my family vacation', name='simulated'),\n",
" HumanMessage(content=\"Sure! I'd be happy to help you plan your family vacation. Can you provide more details about your preferences, such as the destination, budget, duration of the trip, and any specific activities or attractions you have in mind?\"),\n",
" HumanMessage(content=\"Oh, planning family vacations is always a bit of a juggling act, isn't it? We've got a variety of tastes in my family too, so I totally get where you're coming from. We're on a budget, so we usually look for places that won't break the bank. Everyone loves the beach—it's just Aunt Lily who's the odd one out, preferring the mountains.\\n\\nHere's a thought, maybe you can find a coastal area that's near some mountains? That way, the majority of the family gets to enjoy the sand and surf while Aunt Lily isn't too far from a mountain getaway. Depending on where you live, there might be some places not too far away that offer both. \\n\\nFor instance, places like the Central Coast of California have beaches and they're not too far from mountains. Or you could look into a spot like the South of France, if you're up for international travel and can find some deals. I've also heard that places like Costa Rica have both, but I've never been there myself.\\n\\nAs for the budget, I'm always on the lookout for off-season deals or vacation rentals that can accommodate the whole family. It can be way more cost-effective than booking multiple hotel rooms, and you can save a bit by cooking meals at the rental rather than eating out all the time.\\n\\nHave you thought about any specific destinations yet?\", name='simulated'),\n",
" HumanMessage(content=\"Those are great suggestions! Finding a destination that offers both beach and mountain options can be a great compromise for your family. Here are a few more specific destination ideas that might fit your preferences:\\n\\n1. The Oregon Coast, USA: Known for its stunning coastline and nearby mountain ranges like the Cascade Range, the Oregon Coast offers a mix of beautiful beaches, charming coastal towns, and opportunities for hiking in the mountains.\\n\\n2. Bali, Indonesia: This tropical island destination offers gorgeous beaches as well as volcanic mountains like Mount Batur. You can relax on the beach, explore temples, try water sports, and even trek through rice terraces and lush forests.\\n\\n3. Split, Croatia: Located on the stunning Dalmatian Coast, Split offers a mix of beach relaxation and nearby mountain hiking opportunities in places like the Biokovo nature park. Plus, you can explore the historic Old Town and nearby islands like Hvar.\\n\\n4. Cape Town, South Africa: With its iconic Table Mountain and beautiful Atlantic beaches like Camps Bay, Cape Town provides the best of both worlds. You can take a cable car up Table Mountain, visit the penguins at Boulders Beach, and even go on a wine tour in the nearby Cape Winelands.\\n\\nWhen it comes to budget-friendly options, consider booking vacation rentals, researching affordable or all-inclusive resorts, and keeping an eye out for discounts on flights and attractions. It's also advisable to be flexible with your travel dates, as traveling during the offseason can often result in more affordable prices.\\n\\nLet me know if you need any more information or help with planning specific activities or accommodations in any of these destinations!\"),\n",
" HumanMessage(content=\"Oh, those are some fantastic ideas, really! Each of those spots has something unique to offer. I'll definitely have to look into the Oregon Coast. It has that rugged charm, and I've heard it's not as pricey as California. Bali sounds like a dream, honestly, but I have to admit, international travel might be a bit much for the budget this time around. \\n\\nCroatia is one of those places I've always wanted to visit, with all that beautiful coastline and history, but again, might be a stretch budget-wise. Cape Town would be an adventure for sure, but South Africa is a big trip. It's probably out of our range for now.\\n\\nI really appreciate the suggestions about being flexible with travel dates and looking at vacation rentals. That's the kind of approach we usually take. We try to avoid the peak seasons to save some money and find those hidden deals.\\n\\nIt sounds like you've done a fair bit of traveling yourself, or you're just really good at sniffing out the cool spots to visit. Do you travel a lot?\", name='simulated'),\n",
" HumanMessage(content=\"I'm glad you found the suggestions helpful! The Oregon Coast is definitely a more budget-friendly option compared to some other coastal destinations. It offers stunning landscapes, charming towns, and the opportunity to explore both the beach and the mountains.\\n\\nBali is indeed a dream destination, but it's understandable that international travel might not fit within the budget this time. It's always good to keep it in mind for future trips though, as it offers a unique cultural experience along with beautiful beaches and mountains.\\n\\nCroatia is known for its stunning coastline and historic cities like Split and Dubrovnik, but it can sometimes be on the more expensive side. It's always worth checking for deals and considering different accommodation options to make it more affordable.\\n\\nAnd yes, South Africa and Cape Town are definitely big trips. If it's not feasible for the current vacation, you can always keep it on your bucket list for future adventures when the budget allows.\\n\\nAs for your question, I do love to travel and explore different places whenever I get the chance. I'm also always researching and learning about new destinations to be able to offer suggestions and help others plan their trips. It's a passion of mine! If you ever need more help or have any specific questions about the destinations or planning, feel free to ask.\"),\n",
" HumanMessage(content='FINISHED', name='simulated')]"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"gch # These are the message from the final simulation state\n",
"result[\"messages\"]"
]
},
{
"cell_type": "markdown",
"id": "23db8891-5db3-4a98-a283-53d45cd28c60",
"metadata": {},
"source": [
"## Conclusion\n",
"\n",
"In this notebook, you set up a multi-agent simulation to review how your chat bot behaves with simulated users.\n",
"\n",
"To implement this for your chat bot, you can create a dataset of user profiles and questions your chat bot should handle and run periodically. You can use an LLM-as-judge to give the bot an initial score and then manually review to spot check. \n",
"\n",
"LangGraph gives you full control over the simulation so you can manually change the simulated user and the conversation dynamics."
"for chunk in simulation.stream([]):\n",
" # Print out all events aside from the final end chunk\n",
" if END not in chunk:\n",
" print(chunk)\n",
" print(\"----\")"
]
},
{
@@ -401,7 +417,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.2"
"version": "3.11.1"
}
},
"nbformat": 4,
langgraph/graph/__init__.py
+2 -1
View File
@@ -1,4 +1,5 @@
from langgraph.graph.graph import END, Graph
from langgraph.graph.message import MessageGraph
from langgraph.graph.state import StateGraph
__all__ = ["END", "Graph", "StateGraph"]
__all__ = ["END", "Graph", "StateGraph", "MessageGraph"]