From 885d84e4ae68cbdb963e5b5dd4450185135fdff4 Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Thu, 18 Jan 2024 18:17:56 -0800 Subject: [PATCH] Agent simulation --- .../agent-simulation-evaluation.ipynb | 447 ++++++++++++++++++ 1 file changed, 447 insertions(+) create mode 100644 examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb diff --git a/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb b/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb new file mode 100644 index 000000000..fd17e3481 --- /dev/null +++ b/examples/advanced_agents/multi-agent/agent-simulation-evaluation.ipynb @@ -0,0 +1,447 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a3e3ebc4-57af-4fe4-bdd3-36aff67bf276", + "metadata": {}, + "source": [ + "## Agent Evaluation\n", + "\n", + "When building a chat bot, it can be hard to evaluate the multi-turn interactions.\n", + "\n", + "One way to evaluate is by simulating a conversation with a user, then evaluating the outpucomes and trajectory.\n", + "\n", + "Below is an example of how to do this in langgraph." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "0d30b6f7-3bec-4d9f-af50-43dfdc81ae6c", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture --no-stderr\n", + "%pip install -U langchain langchain_openai langsmith pandas" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "30c2f3de-c730-4aec-85a6-af2c2f058803", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "import uuid\n", + "\n", + "\n", + "def _set_if_undefined(var: str):\n", + " if not os.environ.get(var):\n", + " os.environ[var] = getpass(f\"Please provide your {var}\")\n", + "\n", + "\n", + "_set_if_undefined(\"OPENAI_API_KEY\")\n", + "_set_if_undefined(\"LANGCHAIN_API_KEY\")\n", + "_set_if_undefined(\"TAVILY_API_KEY\")\n", + "\n", + "# Optional, add tracing in LangSmith.\n", + "# This will help you visualize and debug the control flow\n", + "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n", + "os.environ[\"LANGCHAIN_PROJECT\"] = \"Agent Simulation Evaluation\"" + ] + }, + { + "cell_type": "markdown", + "id": "321312b4-a1f0-4454-a481-fdac4e37cb7d", + "metadata": {}, + "source": [ + "## Make a user proxy \n", + "\n", + "Really unstructured right now. Will have to think about the UX of this further." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "74bbe7c1-9e62-4bfe-9391-5b1a6bb4e6dc", + "metadata": {}, + "outputs": [], + "source": [ + "# Create a human proxy\n", + "import operator\n", + "from typing import Annotated, List, TypedDict\n", + "\n", + "from langchain_core.messages import AIMessage, BaseMessage, HumanMessage\n", + "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n", + "\n", + "from langgraph.graph import END, StateGraph\n", + "\n", + "\n", + "def create_user_proxy(llm):\n", + " # We would likely want to provide more system prompt customization here\n", + " prompt = ChatPromptTemplate.from_messages(\n", + " [\n", + " (\n", + " \"system\",\n", + " \"You are role-playing the human character: {name}. \"\n", + " \"You are not an AI assistant and you are not supposed to help or assist.\"\n", + " \" You must behave as this human would throughout the conversation below.\"\n", + " \" Your response will be labeled as a human message.\"\n", + " \" You will be evaluated based on how realistic your impersonation of\"\n", + " \" this character is. Here are the details for your character, {name}:\\n\"\n", + " \"{system_prompt}\"\n", + " '\\n\\nWhen you are finished with the conversation, respond with a single word \"FINISHED\"',\n", + " ),\n", + " MessagesPlaceholder(variable_name=\"messages\"),\n", + " ]\n", + " )\n", + " return prompt | llm | (lambda x: {\"messages\": [HumanMessage(content=x.content)]})" + ] + }, + { + "cell_type": "markdown", + "id": "58480cbf-33dc-48ea-85a4-a4436e3fc283", + "metadata": {}, + "source": [ + "## Define the simulation constructor\n", + "\n", + "Construct the graph that connects the user proxy and the evaluated model." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "cc93d114-fa65-4021-a67e-b1e6d4edc88a", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Callable\n", + "\n", + "from langchain_core.runnables import chain\n", + "from langchain_openai import ChatOpenAI\n", + "\n", + "\n", + "class Environment(TypedDict):\n", + " messages: Annotated[List[BaseMessage], operator.add]\n", + " system_prompt: str\n", + " name: str\n", + " input: str\n", + "\n", + "\n", + "def should_continue(state: Environment):\n", + " if state[\"messages\"][-1].content.strip().endswith(\"FINISHED\"):\n", + " return \"end\"\n", + " return \"continue\"\n", + "\n", + "\n", + "@chain\n", + "def next_message(state: Environment):\n", + " return {\"input\": state[\"messages\"][-1].content}\n", + "\n", + "\n", + "def enter(inputs: dict):\n", + " inputs[\"messages\"] = [HumanMessage(content=inputs[\"input\"], name=inputs[\"name\"])]\n", + " return inputs\n", + "\n", + "\n", + "def exit(agent_output):\n", + " # If we do an ai message here, the user proxy llm\n", + " # will usually forget it's acting.\n", + " return {\"messages\": [HumanMessage(content=agent_output[\"output\"])]}\n", + "\n", + "\n", + "def create_simulation(user_proxy_llm, agent_constructor: Callable):\n", + " user_proxy = create_user_proxy(user_proxy_llm)\n", + " agent_to_evaluate = agent_constructor()\n", + "\n", + " graph_builder = StateGraph(Environment)\n", + " graph_builder.add_node(\"user\", user_proxy)\n", + " # give it a single string input\n", + " graph_builder.add_node(\"agent\", next_message | agent_to_evaluate | exit)\n", + " graph_builder.add_edge(\"agent\", \"user\")\n", + " graph_builder.add_conditional_edges(\n", + " \"user\",\n", + " should_continue,\n", + " {\n", + " \"end\": END,\n", + " \"continue\": \"agent\",\n", + " },\n", + " )\n", + " graph_builder.set_entry_point(\"agent\")\n", + " return (enter | graph_builder.compile()).with_config(run_name=\"Agent Simulation\")" + ] + }, + { + "cell_type": "markdown", + "id": "243d07dd-187c-4cc7-815d-dee4fbe84df4", + "metadata": {}, + "source": [ + "## Create your agent\n", + "\n", + "We make a constructor. There's some additional data munging to make the simulator output work with your model." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c462eac5-53b4-4ea3-b38a-40940826df6b", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_core.output_parsers import StrOutputParser\n", + "from langchain_core.runnables.history import RunnableWithMessageHistory\n", + "\n", + "\n", + "# This returns any runnable. It would probably accept \"input\" or something\n", + "# It's the agent you want to evaluate\n", + "def creat_conversational_agent():\n", + " messages = []\n", + "\n", + " def add_user_message(inputs):\n", + " messages.append(HumanMessage(content=inputs[\"input\"]))\n", + " return {\"messages\": messages}\n", + "\n", + " def add_ai_message(output):\n", + " messages.append(output)\n", + " return {\"output\": output.content}\n", + "\n", + " chain = (\n", + " add_user_message\n", + " | ChatPromptTemplate.from_messages(\n", + " [\n", + " (\"system\", \"you are a helpful assistant\"),\n", + " MessagesPlaceholder(variable_name=\"messages\"),\n", + " ]\n", + " )\n", + " | ChatOpenAI(model=\"gpt-3.5-turbo\")\n", + " | add_ai_message\n", + " )\n", + " return chain" + ] + }, + { + "cell_type": "markdown", + "id": "219859c4-3785-4680-8738-9512a64be3bf", + "metadata": {}, + "source": [ + "## Create a dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "394c19df-05fa-47df-8e18-a86c0bc4e138", + "metadata": {}, + "outputs": [], + "source": [ + "import langsmith\n", + "\n", + "client = langsmith.Client()\n", + "\n", + "examples = [\n", + " (\n", + " \"Adam\",\n", + " \"You really needs to find the restroom. You only speak English.\",\n", + " \"Hi there, where's the restroom?\",\n", + " ),\n", + " (\n", + " \"Steve\",\n", + " \"You are a developer who needs help with an error.\",\n", + " \"How do I fix this? NameError: module terminator cannot be found\",\n", + " ),\n", + " (\"Jezebel\", \"You want to learn more about langchain\", \"So what's langchain?\"),\n", + "]\n", + "\n", + "dataset_name = \"Agent Simulation\"\n", + "if not client.has_dataset(dataset_name=dataset_name):\n", + " ds = client.create_dataset(dataset_name=dataset_name)\n", + " client.create_examples(\n", + " inputs=[\n", + " {\"system_prompt\": sys, \"name\": name, \"input\": input}\n", + " for name, sys, input in examples\n", + " ],\n", + " dataset_id=ds.id,\n", + " )\n", + "# else:\n", + "# client.delete_dataset(dataset_name=dataset_name)" + ] + }, + { + "cell_type": "markdown", + "id": "990f198d-29c9-400b-b7e8-c6c5a0b7ef77", + "metadata": {}, + "source": [ + "## Define evaluators\n", + "\n", + "We will use an LLM as judge here. In reality, you would probably add additional\n", + "context to the ground truth to make the results more reliable." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "d8b1ebd8-60c5-4051-85d8-2576c81b3b41", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.load import load\n", + "from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser\n", + "from langchain_core.pydantic_v1 import BaseModel, Field\n", + "from langsmith.evaluation import run_evaluator\n", + "\n", + "\n", + "class GradeConversation(BaseModel):\n", + " \"\"\"Grade the conversation based on quality.\"\"\"\n", + "\n", + " reasoning: Annotated[\n", + " str, \"The step-by-step reasoning explaining the conversation quality score\"\n", + " ]\n", + " score: Annotated[int, \"The score on a scale from 0 to 5\"] = Field(gte=0, lte=5)\n", + "\n", + "\n", + "@run_evaluator\n", + "def conversation_quality(run, example):\n", + " if not run.outputs:\n", + " return {\"score\": 0, \"comment\": \"Unfinished\"}\n", + " # TODO: Incorporate examples\n", + " prompt = ChatPromptTemplate.from_messages(\n", + " [\n", + " (\n", + " \"system\",\n", + " \"You are grading the quality of the following conversation.\"\n", + " \"Submit your response using the GradeConversation function.\",\n", + " ),\n", + " MessagesPlaceholder(variable_name=\"messages\"),\n", + " (\"system\", \"Review the conversation above and submit your score.\"),\n", + " ]\n", + " )\n", + " bound = ChatOpenAI(model=\"gpt-4-1106-preview\").bind_functions(\n", + " functions=[GradeConversation], function_call=\"GradeConversation\"\n", + " )\n", + " chain = prompt | bound | JsonOutputFunctionsParser()\n", + " result = chain.invoke(load(run.outputs))\n", + " score = result[\"score\"] / 5\n", + " return {\"score\": score, \"comment\": result[\"reasoning\"]}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "39d6287f-2768-42f5-a428-d753ece6048a", + "metadata": {}, + "outputs": [], + "source": [ + "# llm = ChatOpenAI(model=\"gpt-4-1106-preview\")\n", + "# graph = create_simulation(llm, creat_conversational_agent)\n", + "\n", + "# graph.invoke(\n", + "# {\n", + "# \"system_prompt\": \"You really needs to find the restroom. You only speak English.\",\n", + "# \"name\": \"Adam\",\n", + "# \"input\": \"I really need to go\",\n", + "# }\n", + "# )" + ] + }, + { + "cell_type": "markdown", + "id": "ed54f4c9-e52d-421f-935d-02aab5d5d6cb", + "metadata": {}, + "source": [ + "## Run Evaluation" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "be206282-2d7c-4f5c-9c71-77dd9e192b6d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/wfh/.pyenv/versions/3.11.2/lib/python3.11/site-packages/langchain/smith/evaluation/runner_utils.py:1244: LangChainDeprecationWarning: The following arguments are deprecated and will be removed in a future release: dict_keys(['revision_id']).\n", + " warn_deprecated(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "View the evaluation results for project 'ample-color-99' at:\n", + "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/e5fde795-82eb-4ca2-8f2e-9afa666b5c0f/compare?selectedSessions=61f4e55a-1496-49e7-8ef3-e39376a64d8a\n", + "\n", + "View all tests for Dataset Agent Simulation at:\n", + "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/e5fde795-82eb-4ca2-8f2e-9afa666b5c0f\n", + "[> ] 0/3" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/wfh/code/lc/langchain/libs/core/langchain_core/_api/beta_decorator.py:86: LangChainBetaWarning: The function `load` is in beta. It is actively being worked on, so the API may change.\n", + " warn_beta(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[------------------------------------------------->] 3/3" + ] + } + ], + "source": [ + "import functools\n", + "\n", + "from langchain.smith import RunEvalConfig\n", + "\n", + "config = RunEvalConfig(\n", + " custom_evaluators=[conversation_quality],\n", + ")\n", + "results = client.run_on_dataset(\n", + " dataset_name=dataset_name,\n", + " llm_or_chain_factory=functools.partial(\n", + " create_simulation,\n", + " ChatOpenAI(model=\"gpt-4-1106-preview\"),\n", + " creat_conversational_agent,\n", + " ),\n", + " evaluation=config,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56a40a17-abba-4f12-800a-945a81bc6fa1", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}