From 2fd49260618e5659c3222cd463f5806ac028145f Mon Sep 17 00:00:00 2001 From: Lance Martin Date: Wed, 21 Feb 2024 14:27:54 -0800 Subject: [PATCH] Add evals --- .../lcel-teacher-langgraph.ipynb | 189 ++++++++++++++++-- 1 file changed, 170 insertions(+), 19 deletions(-) diff --git a/examples/code_assistant/lcel-teacher-langgraph.ipynb b/examples/code_assistant/lcel-teacher-langgraph.ipynb index f4c7170bb..1f7a3dcd8 100644 --- a/examples/code_assistant/lcel-teacher-langgraph.ipynb +++ b/examples/code_assistant/lcel-teacher-langgraph.ipynb @@ -183,6 +183,7 @@ " state_dict = state[\"keys\"]\n", " question = state_dict[\"question\"]\n", " docs = state_dict[\"docs\"]\n", + " iter = state_dict[\"iterations\"]\n", " \n", " ## Data model\n", " class code(BaseModel):\n", @@ -279,8 +280,12 @@ " )\n", "\n", " code_solution = chain.invoke({\"question\":question})\n", + "\n", + " iter = iter+1 \n", + " print(\"Iterations!\")\n", + " print(iter)\n", " \n", - " return {\"keys\": {\"generation\": code_solution, \"question\": question, \"docs\": docs}}\n", + " return {\"keys\": {\"generation\": code_solution, \"question\": question, \"docs\": docs, \"iterations\":iter}}\n", "\n", "def check_code_imports(state):\n", " \"\"\"\n", @@ -300,6 +305,7 @@ " docs = state_dict[\"docs\"]\n", " code_solution = state_dict[\"generation\"]\n", " imports = code_solution[0].imports\n", + " iter = state_dict[\"iterations\"]\n", "\n", " try: \n", " # Attempt to execute the imports\n", @@ -316,7 +322,7 @@ " # No errors occurred\n", " error = \"None\"\n", "\n", - " return {\"keys\": {\"generation\": code_solution, \"question\": question, \"error\": error, \"docs\": docs}}\n", + " return {\"keys\": {\"generation\": code_solution, \"question\": question, \"error\": error, \"docs\": docs, \"iterations\":iter}}\n", "\n", "def check_code_execution(state):\n", " \"\"\"\n", @@ -339,6 +345,7 @@ " imports = code_solution[0].imports\n", " code = code_solution[0].code\n", " code_block = imports +\"\\n\"+ code\n", + " iter = state_dict[\"iterations\"]\n", "\n", " try: \n", " # Attempt to execute the code block\n", @@ -361,6 +368,7 @@ " \"docs\": docs,\n", " \"prefix\":prefix,\n", " \"imports\":imports,\n", + " \"iterations\":iter,\n", " \"code\":code}}\n", "\n", "### Edges\n", @@ -394,7 +402,7 @@ "\n", "def decide_to_finish(state):\n", " \"\"\"\n", - " Determines whether to finish.\n", + " Determines whether to finish (re-try code 3 times.\n", "\n", " Args:\n", " state (dict): The current graph state\n", @@ -408,8 +416,9 @@ " question = state_dict[\"question\"]\n", " code_solution = state_dict[\"generation\"]\n", " error = state_dict[\"error\"]\n", + " iter = state_dict[\"iterations\"]\n", "\n", - " if error == \"None\":\n", + " if error == \"None\" or iter == 3:\n", " # All documents have been filtered check_relevance\n", " # We will re-generate a new query\n", " print(\"---DECISION: TEST CODE EXECUTION---\")\n", @@ -472,6 +481,92 @@ "Let's create a LangSmith evaluator [here](https://docs.smith.langchain.com/evaluation/faq/custom-evaluators) to test each." ] }, + { + "cell_type": "markdown", + "id": "86411645-98f8-4d19-889f-c78f3c026380", + "metadata": {}, + "source": [ + "### Base Case RAG" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d73dfd70-d266-4be7-9fd7-ed1f5cd432c6", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_core.runnables import RunnableLambda\n", + "\n", + "## Data model\n", + "class code(BaseModel):\n", + " \"\"\"Code output\"\"\"\n", + " prefix: str = Field(description=\"Description of the problem and approach\")\n", + " imports: str = Field(description=\"Code block import statements\")\n", + " code: str = Field(description=\"Code block not including import statements\")\n", + "\n", + "## LLM\n", + "model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n", + "\n", + "# Tool\n", + "code_tool_oai = convert_to_openai_tool(code)\n", + "\n", + "# LLM with tool and enforce invocation\n", + "llm_with_tool = model.bind(\n", + " tools=[convert_to_openai_tool(code_tool_oai)],\n", + " tool_choice={\"type\": \"function\", \"function\": {\"name\": \"code\"}},\n", + ")\n", + "\n", + "# Parser\n", + "parser_tool = PydanticToolsParser(tools=[code])\n", + "\n", + "# Create a prompt template with format instructions and the query\n", + "prompt = PromptTemplate(\n", + " template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n", + " Here is a full set of LCEL documentation: \n", + " \\n ------- \\n\n", + " {context} \n", + " \\n ------- \\n\n", + " Answer the user question based on the above provided documentation. \\n\n", + " Ensure any code you provide can be executed with all required imports and variables defined. \\n\n", + " Structure your answer with a description of the code solution. \\n\n", + " Then list the imports. And finally list the functioning code block. \\n\n", + " Here is the user question: \\n --- --- --- \\n {question}\"\"\",\n", + " input_variables=[\"question\",\"context\"])\n", + "\n", + "def parse_answer_to_dict(x):\n", + " return x[0].dict()\n", + "\n", + "chain_base_rag = (\n", + " {\n", + " \"context\": lambda x: concatenated_content,\n", + " \"question\": RunnablePassthrough(),\n", + " }\n", + " | prompt\n", + " | llm_with_tool\n", + " | parser_tool\n", + " | RunnableLambda(parse_answer_to_dict)\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a980b398-ad4c-4b21-8e6d-77ab03130526", + "metadata": {}, + "outputs": [], + "source": [ + "answer = chain_base_rag.invoke(\"How can I write a RAG chain?\")" + ] + }, + { + "cell_type": "markdown", + "id": "2901bdef-a78c-4831-81f2-927c35503fd9", + "metadata": {}, + "source": [ + "### Eval w/o LangGraph" + ] + }, { "cell_type": "code", "execution_count": null, @@ -479,12 +574,76 @@ "metadata": {}, "outputs": [], "source": [ + "### No LangGraph\n", + "\n", + "import uuid\n", + "from langsmith import Client\n", + "from langchain.smith import RunEvalConfig, run_on_dataset\n", "from langsmith.evaluation import EvaluationResult, run_evaluator\n", "from langsmith.schemas import Example, Run\n", "from typing import Union\n", "\n", "@run_evaluator\n", "def check_import(run: Run, example: Union[Example, None] = None):\n", + " model_outputs = run.outputs\n", + " imports = model_outputs['imports']\n", + " try:\n", + " exec(imports)\n", + " score = 1\n", + " except:\n", + " score = 0\n", + " return EvaluationResult(key=\"check_import\", score=score)\n", + "\n", + "@run_evaluator\n", + "def check_execution(run: Run, example: Union[Example, None] = None):\n", + " model_outputs = run.outputs\n", + " imports = model_outputs['imports']\n", + " code = model_outputs['code']\n", + " code_to_execute = imports +\"\\n\"+ code\n", + " try:\n", + " exec(code_to_execute)\n", + " score = 1\n", + " except:\n", + " score = 0\n", + " return EvaluationResult(key=\"check_execution\", score=score)\n", + "\n", + "# Config\n", + "evaluation_config = RunEvalConfig(\n", + " custom_evaluators = [check_import,check_execution],\n", + ")\n", + "\n", + "client = Client()\n", + "\n", + "# Run eval on base chain\n", + "run_id = uuid.uuid4().hex[:4]\n", + "project_name = \"context-stuffing-no-langgraph\"\n", + "client.run_on_dataset(\n", + " dataset_name=\"lcel-teacher-eval\",\n", + " llm_or_chain_factory= lambda: (lambda x: x[\"question\"]) | chain_base_rag,\n", + " evaluation=evaluation_config,\n", + " project_name=f\"{run_id}-{project_name}\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "50742bc3-7351-414e-942b-e40bb258ddff", + "metadata": {}, + "source": [ + "### Eval w/ LangGraph" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "959b473a-15d4-4b61-a319-32bd33f6a7fb", + "metadata": {}, + "outputs": [], + "source": [ + "### LangGraph\n", + "\n", + "@run_evaluator\n", + "def check_import(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs[\"keys\"]\n", " imports = model_outputs['imports']\n", " try:\n", @@ -509,32 +668,24 @@ " except:\n", " score = 0\n", " print(\"Score: 0!\")\n", - " return EvaluationResult(key=\"check_execution\", score=score)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "959b473a-15d4-4b61-a319-32bd33f6a7fb", - "metadata": {}, - "outputs": [], - "source": [ - "from langsmith import Client\n", - "from langchain.smith import RunEvalConfig, run_on_dataset\n", + " return EvaluationResult(key=\"check_execution\", score=score)\n", "\n", + "# Config\n", "evaluation_config = RunEvalConfig(\n", " custom_evaluators = [check_import,check_execution],\n", ")\n", "\n", - "client = Client()\n", - "\n", + "config = {\"recursion_limit\": 50}\n", "def model(input):\n", - " return app.invoke({\"keys\":{**input, \"docs\": concatenated_content}})\n", + " return app.invoke({\"keys\":{**input, \"docs\": concatenated_content, \"iterations\":0}},config=config)\n", "\n", + "run_id = uuid.uuid4().hex[:4]\n", + "project_name = \"context-stuffing-with-langgraph\"\n", "client.run_on_dataset(\n", " dataset_name=\"lcel-teacher-eval\",\n", " llm_or_chain_factory=model,\n", " evaluation=evaluation_config,\n", + " project_name=f\"{run_id}-{project_name}\",\n", ")" ] },