mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-08-29 03:09:45 +02:00
20 KiB
20 KiB
Chat Bot Evaluation as Multi-agent Simulation
When building a chat bot, such as a customer support assistant, it can be hard to properly evalute your bot's performance. It's time-consuming to have to manually interact with it intensively for each code change.
One way to make the evaluation process easier and more reproducible is to simulate a user interaction.
With LangGraph, it's easy to set this up. Below is an example of how to create a "virtual user" to simulate a conversation.
The overall simulation looks something like this:
First, we'll set up our environment.
In [1]:
# %%capture --no-stderr
# %pip install -U langgraph langchain langchain_openaiIn [2]:
import getpass
import os
import uuid
def _set_if_undefined(var: str):
if not os.environ.get(var):
os.environ[var] = getpass(f"Please provide your {var}")
_set_if_undefined("OPENAI_API_KEY")
_set_if_undefined("LANGCHAIN_API_KEY")
# Optional, add tracing in LangSmith.
# This will help you visualize and debug the control flow
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGCHAIN_PROJECT"] = "Agent Simulation Evaluation"In [3]:
from typing import List
import openai
# This is flexible, but you can define your agent here, or call your agent API here.
def my_chat_bot(messages: List[dict]) -> dict:
completion = openai.chat.completions.create(
messages=messages, model="gpt-3.5-turbo"
)
return completion.choices[0].message.model_dump()In [12]:
import operator
from typing import Annotated, Callable, Dict, List, TypedDict
from langchain.adapters.openai import convert_message_to_dict
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage
from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
from langchain_core.runnables import chain
from langchain_openai import ChatOpenAI
from langgraph.graph import END, StateGraph
SIMULATED_USER_NAME = "simulated"
# This is just an example, we can
# configure additional parameters if
# you want more control
class SimulatedUserConfig(TypedDict):
system_prompt: str
# This is the input to every node in the simulation graph
# It tracks the graph state over time. Our only "state"
# is the conversation messages, while the user config
# is provided to make the virtual user more unique or realistic
class Environment(TypedDict):
messages: Annotated[List[BaseMessage], operator.add]
simulated_user_config: SimulatedUserConfig
# We currently let the virtual user decide if the conversation can end
# we could also track max conversation turns, add a conversation "supervisor"
# or use other heuristics to control the dialogue flow
def should_continue(state: Environment):
"""Determine if the simulation should continue."""
if state["messages"][-1].content.strip().endswith("FINISHED"):
return "end"
return "continue"
## The next two functions define the API between the simulation
# and the chat bot you wish to test.
# We are assuming your chat bot accepts a list of OAI messages
@chain
def get_messages_for_agent(state: Environment):
"""Convert the simulation state to the input
for your agent you want to evaluate."""
return [convert_message_to_dict(message) for message in state["messages"]]
# This takes the output of your chat bot
# and adds it to the simulation state
def get_response_message_from_agent(agent_output):
"""Get the response from the agent you are evaluting,
and use it to update the simulation state."""
# If we do an ai message here, the user proxy llm
# will usually forget it's acting.
return {"messages": [HumanMessage(content=agent_output["content"])]}
# This is run once at the beginning of the simulation.
# It's more convenient to just write an input string
# than to pass in a full message, but this could be removed below
def enter(inputs: dict):
"""Start the simulation. This makes it less verbose to invoke."""
inputs["messages"] = [
HumanMessage(content=inputs["input"], name=SIMULATED_USER_NAME)
]
return inputs
def create_simulation(chat_bot: Callable[[List[Dict]], Dict], simulated_user_llm=None):
"""Create a chat bot simulation graph.
Args:
- chat_bot: the agent you are evaluating. Accepts a list of openai messages
and returns an openai assistant message
- simulated_user_llm: the LLM to power your virtual user.
Defaults to gpt-4-1106-preview
Returns:
- simulation: an runnable object formed from compiling the state graph
"""
# This defines the virtual user proxy
prompt = ChatPromptTemplate.from_messages(
[
(
"system",
"You are role-playing a human character: '{name}'. "
"You are not an AI assistant and you are not supposed to help or assist."
" You must behave as this human would throughout the conversation below.\n\n"
"Your messages will bear the name 'simulated', but DO NOT under any circumstances"
"say that you are 'simulated'. You will be evaluated based on how realistic your"
"impersonation of this character is. This must feel real! Here are the details for your character:"
"\n"
"{system_prompt}" # This is the value you provide to characterize the user
'\n\nWhen you are finished with the conversation, respond with a single word "FINISHED"',
),
MessagesPlaceholder(variable_name="messages"),
]
).partial(name=SIMULATED_USER_NAME)
simulated_user_llm = simulated_user_llm or ChatOpenAI(model="gpt-4-1106-preview")
user_proxy = (
(lambda x: {**x, **x["simulated_user_config"]})
| prompt
| simulated_user_llm
| (
lambda x: {
"messages": [HumanMessage(content=x.content, name=SIMULATED_USER_NAME)]
}
)
)
graph_builder = StateGraph(Environment)
graph_builder.add_node("user", user_proxy)
graph_builder.add_node(
# The "|" syntax composes these steps in the pipeline to map between
# the simulation state and your chat bot's API
"chat_bot",
get_messages_for_agent | chat_bot | get_response_message_from_agent,
)
# Every response from your chat bot will automatically go to the
# simulated user
graph_builder.add_edge("chat_bot", "user")
graph_builder.add_conditional_edges(
"user",
should_continue,
# If the finish criteria are met, we will stop the simulation,
# otherwise, the virtual user's message will be sent to your chat bot
{
"end": END,
"continue": "chat_bot",
},
)
# The input will first go to your chat bot
graph_builder.set_entry_point("chat_bot")
return (enter | graph_builder.compile()).with_config(run_name="Agent Simulation")In [13]:
simulation = create_simulation(my_chat_bot)In [14]:
from langchain_core.tracers.context import tracing_v2_enabled
# The tracing context manager lets us easily fetch the trace URL in-context.
# You can turn this off if you don't want to trace the execution.
with tracing_v2_enabled() as tracer:
result = simulation.invoke(
{
"simulated_user_config": {
"system_prompt": "You are on a budget. Your family is hard to please."
" They all like the beach, except for Aunt Lily, who prefers the mountains."
},
"input": "help me plan my family vacation",
}
)
# You can go to this run to review the entire simulation trace
url = tracer.get_run_url()Skipping write for channel input which has no readers
Review Result
If you've traced the run, you can see the resulting trace in the UI by clicking on the url. Select the last 'ChatOpenAI' call in the trace to see the full conversation in a single view.
From this run, you can manually annotate it to score its quality.
In [ ]:
urlIn [15]:
result["messages"]Out [15]:
[HumanMessage(content='help me plan my family vacation', name='simulated'), HumanMessage(content="Sure! I'd be happy to help you plan your family vacation. Can you provide more details about your preferences, such as the destination, budget, duration of the trip, and any specific activities or attractions you have in mind?"), HumanMessage(content="Oh, planning family vacations is always a bit of a juggling act, isn't it? We've got a variety of tastes in my family too, so I totally get where you're coming from. We're on a budget, so we usually look for places that won't break the bank. Everyone loves the beach—it's just Aunt Lily who's the odd one out, preferring the mountains.\n\nHere's a thought, maybe you can find a coastal area that's near some mountains? That way, the majority of the family gets to enjoy the sand and surf while Aunt Lily isn't too far from a mountain getaway. Depending on where you live, there might be some places not too far away that offer both. \n\nFor instance, places like the Central Coast of California have beaches and they're not too far from mountains. Or you could look into a spot like the South of France, if you're up for international travel and can find some deals. I've also heard that places like Costa Rica have both, but I've never been there myself.\n\nAs for the budget, I'm always on the lookout for off-season deals or vacation rentals that can accommodate the whole family. It can be way more cost-effective than booking multiple hotel rooms, and you can save a bit by cooking meals at the rental rather than eating out all the time.\n\nHave you thought about any specific destinations yet?", name='simulated'), HumanMessage(content="Those are great suggestions! Finding a destination that offers both beach and mountain options can be a great compromise for your family. Here are a few more specific destination ideas that might fit your preferences:\n\n1. The Oregon Coast, USA: Known for its stunning coastline and nearby mountain ranges like the Cascade Range, the Oregon Coast offers a mix of beautiful beaches, charming coastal towns, and opportunities for hiking in the mountains.\n\n2. Bali, Indonesia: This tropical island destination offers gorgeous beaches as well as volcanic mountains like Mount Batur. You can relax on the beach, explore temples, try water sports, and even trek through rice terraces and lush forests.\n\n3. Split, Croatia: Located on the stunning Dalmatian Coast, Split offers a mix of beach relaxation and nearby mountain hiking opportunities in places like the Biokovo nature park. Plus, you can explore the historic Old Town and nearby islands like Hvar.\n\n4. Cape Town, South Africa: With its iconic Table Mountain and beautiful Atlantic beaches like Camps Bay, Cape Town provides the best of both worlds. You can take a cable car up Table Mountain, visit the penguins at Boulders Beach, and even go on a wine tour in the nearby Cape Winelands.\n\nWhen it comes to budget-friendly options, consider booking vacation rentals, researching affordable or all-inclusive resorts, and keeping an eye out for discounts on flights and attractions. It's also advisable to be flexible with your travel dates, as traveling during the offseason can often result in more affordable prices.\n\nLet me know if you need any more information or help with planning specific activities or accommodations in any of these destinations!"), HumanMessage(content="Oh, those are some fantastic ideas, really! Each of those spots has something unique to offer. I'll definitely have to look into the Oregon Coast. It has that rugged charm, and I've heard it's not as pricey as California. Bali sounds like a dream, honestly, but I have to admit, international travel might be a bit much for the budget this time around. \n\nCroatia is one of those places I've always wanted to visit, with all that beautiful coastline and history, but again, might be a stretch budget-wise. Cape Town would be an adventure for sure, but South Africa is a big trip. It's probably out of our range for now.\n\nI really appreciate the suggestions about being flexible with travel dates and looking at vacation rentals. That's the kind of approach we usually take. We try to avoid the peak seasons to save some money and find those hidden deals.\n\nIt sounds like you've done a fair bit of traveling yourself, or you're just really good at sniffing out the cool spots to visit. Do you travel a lot?", name='simulated'), HumanMessage(content="I'm glad you found the suggestions helpful! The Oregon Coast is definitely a more budget-friendly option compared to some other coastal destinations. It offers stunning landscapes, charming towns, and the opportunity to explore both the beach and the mountains.\n\nBali is indeed a dream destination, but it's understandable that international travel might not fit within the budget this time. It's always good to keep it in mind for future trips though, as it offers a unique cultural experience along with beautiful beaches and mountains.\n\nCroatia is known for its stunning coastline and historic cities like Split and Dubrovnik, but it can sometimes be on the more expensive side. It's always worth checking for deals and considering different accommodation options to make it more affordable.\n\nAnd yes, South Africa and Cape Town are definitely big trips. If it's not feasible for the current vacation, you can always keep it on your bucket list for future adventures when the budget allows.\n\nAs for your question, I do love to travel and explore different places whenever I get the chance. I'm also always researching and learning about new destinations to be able to offer suggestions and help others plan their trips. It's a passion of mine! If you ever need more help or have any specific questions about the destinations or planning, feel free to ask."), HumanMessage(content='FINISHED', name='simulated')]
In [ ]: