WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-09-07 02:07:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add evals

Browse Source
This commit is contained in:
Lance Martin
2024-02-21 14:27:54 -08:00
parent a902b3c19b
commit 2fd4926061
1 changed files with 170 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/code_assistant/lcel-teacher-langgraph.ipynb
+170 -19
View File
@@ -183,6 +183,7 @@
" state_dict = state[\"keys\"]\n",
" question = state_dict[\"question\"]\n",
" docs = state_dict[\"docs\"]\n",
" iter = state_dict[\"iterations\"]\n",
" \n",
" ## Data model\n",
" class code(BaseModel):\n",
@@ -279,8 +280,12 @@
" )\n",
"\n",
" code_solution = chain.invoke({\"question\":question})\n",
"\n",
" iter = iter+1 \n",
" print(\"Iterations!\")\n",
" print(iter)\n",
" \n",
" return {\"keys\": {\"generation\": code_solution, \"question\": question, \"docs\": docs}}\n",
" return {\"keys\": {\"generation\": code_solution, \"question\": question, \"docs\": docs, \"iterations\":iter}}\n",
"\n",
"def check_code_imports(state):\n",
" \"\"\"\n",
@@ -300,6 +305,7 @@
" docs = state_dict[\"docs\"]\n",
" code_solution = state_dict[\"generation\"]\n",
" imports = code_solution[0].imports\n",
" iter = state_dict[\"iterations\"]\n",
"\n",
" try: \n",
" # Attempt to execute the imports\n",
@@ -316,7 +322,7 @@
" # No errors occurred\n",
" error = \"None\"\n",
"\n",
" return {\"keys\": {\"generation\": code_solution, \"question\": question, \"error\": error, \"docs\": docs}}\n",
" return {\"keys\": {\"generation\": code_solution, \"question\": question, \"error\": error, \"docs\": docs, \"iterations\":iter}}\n",
"\n",
"def check_code_execution(state):\n",
" \"\"\"\n",
@@ -339,6 +345,7 @@
" imports = code_solution[0].imports\n",
" code = code_solution[0].code\n",
" code_block = imports +\"\\n\"+ code\n",
" iter = state_dict[\"iterations\"]\n",
"\n",
" try: \n",
" # Attempt to execute the code block\n",
@@ -361,6 +368,7 @@
" \"docs\": docs,\n",
" \"prefix\":prefix,\n",
" \"imports\":imports,\n",
" \"iterations\":iter,\n",
" \"code\":code}}\n",
"\n",
"### Edges\n",
@@ -394,7 +402,7 @@
"\n",
"def decide_to_finish(state):\n",
" \"\"\"\n",
" Determines whether to finish.\n",
" Determines whether to finish (re-try code 3 times.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
@@ -408,8 +416,9 @@
" question = state_dict[\"question\"]\n",
" code_solution = state_dict[\"generation\"]\n",
" error = state_dict[\"error\"]\n",
" iter = state_dict[\"iterations\"]\n",
"\n",
" if error == \"None\":\n",
" if error == \"None\" or iter == 3:\n",
" # All documents have been filtered check_relevance\n",
" # We will re-generate a new query\n",
" print(\"---DECISION: TEST CODE EXECUTION---\")\n",
@@ -472,6 +481,92 @@
"Let's create a LangSmith evaluator [here](https://docs.smith.langchain.com/evaluation/faq/custom-evaluators) to test each."
]
},
{
"cell_type": "markdown",
"id": "86411645-98f8-4d19-889f-c78f3c026380",
"metadata": {},
"source": [
"### Base Case RAG"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d73dfd70-d266-4be7-9fd7-ed1f5cd432c6",
"metadata": {},
"outputs": [],
"source": [
"from langchain_core.runnables import RunnableLambda\n",
"\n",
"## Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
"\n",
"## LLM\n",
"model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
"\n",
"# Tool\n",
"code_tool_oai = convert_to_openai_tool(code)\n",
"\n",
"# LLM with tool and enforce invocation\n",
"llm_with_tool = model.bind(\n",
" tools=[convert_to_openai_tool(code_tool_oai)],\n",
" tool_choice={\"type\": \"function\", \"function\": {\"name\": \"code\"}},\n",
")\n",
"\n",
"# Parser\n",
"parser_tool = PydanticToolsParser(tools=[code])\n",
"\n",
"# Create a prompt template with format instructions and the query\n",
"prompt = PromptTemplate(\n",
" template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is a full set of LCEL documentation: \n",
" \\n ------- \\n\n",
" {context} \n",
" \\n ------- \\n\n",
" Answer the user question based on the above provided documentation. \\n\n",
" Ensure any code you provide can be executed with all required imports and variables defined. \\n\n",
" Structure your answer with a description of the code solution. \\n\n",
" Then list the imports. And finally list the functioning code block. \\n\n",
" Here is the user question: \\n --- --- --- \\n {question}\"\"\",\n",
" input_variables=[\"question\",\"context\"])\n",
"\n",
"def parse_answer_to_dict(x):\n",
" return x[0].dict()\n",
"\n",
"chain_base_rag = (\n",
" {\n",
" \"context\": lambda x: concatenated_content,\n",
" \"question\": RunnablePassthrough(),\n",
" }\n",
" | prompt\n",
" | llm_with_tool\n",
" | parser_tool\n",
" | RunnableLambda(parse_answer_to_dict)\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a980b398-ad4c-4b21-8e6d-77ab03130526",
"metadata": {},
"outputs": [],
"source": [
"answer = chain_base_rag.invoke(\"How can I write a RAG chain?\")"
]
},
{
"cell_type": "markdown",
"id": "2901bdef-a78c-4831-81f2-927c35503fd9",
"metadata": {},
"source": [
"### Eval w/o LangGraph"
]
},
{
"cell_type": "code",
"execution_count": null,
@@ -479,12 +574,76 @@
"metadata": {},
"outputs": [],
"source": [
"### No LangGraph\n",
"\n",
"import uuid\n",
"from langsmith import Client\n",
"from langchain.smith import RunEvalConfig, run_on_dataset\n",
"from langsmith.evaluation import EvaluationResult, run_evaluator\n",
"from langsmith.schemas import Example, Run\n",
"from typing import Union\n",
"\n",
"@run_evaluator\n",
"def check_import(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs\n",
" imports = model_outputs['imports']\n",
" try:\n",
" exec(imports)\n",
" score = 1\n",
" except:\n",
" score = 0\n",
" return EvaluationResult(key=\"check_import\", score=score)\n",
"\n",
"@run_evaluator\n",
"def check_execution(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs\n",
" imports = model_outputs['imports']\n",
" code = model_outputs['code']\n",
" code_to_execute = imports +\"\\n\"+ code\n",
" try:\n",
" exec(code_to_execute)\n",
" score = 1\n",
" except:\n",
" score = 0\n",
" return EvaluationResult(key=\"check_execution\", score=score)\n",
"\n",
"# Config\n",
"evaluation_config = RunEvalConfig(\n",
" custom_evaluators = [check_import,check_execution],\n",
")\n",
"\n",
"client = Client()\n",
"\n",
"# Run eval on base chain\n",
"run_id = uuid.uuid4().hex[:4]\n",
"project_name = \"context-stuffing-no-langgraph\"\n",
"client.run_on_dataset(\n",
" dataset_name=\"lcel-teacher-eval\",\n",
" llm_or_chain_factory= lambda: (lambda x: x[\"question\"]) | chain_base_rag,\n",
" evaluation=evaluation_config,\n",
" project_name=f\"{run_id}-{project_name}\",\n",
")"
]
},
{
"cell_type": "markdown",
"id": "50742bc3-7351-414e-942b-e40bb258ddff",
"metadata": {},
"source": [
"### Eval w/ LangGraph"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "959b473a-15d4-4b61-a319-32bd33f6a7fb",
"metadata": {},
"outputs": [],
"source": [
"### LangGraph\n",
"\n",
"@run_evaluator\n",
"def check_import(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs[\"keys\"]\n",
" imports = model_outputs['imports']\n",
" try:\n",
@@ -509,32 +668,24 @@
" except:\n",
" score = 0\n",
" print(\"Score: 0!\")\n",
" return EvaluationResult(key=\"check_execution\", score=score)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "959b473a-15d4-4b61-a319-32bd33f6a7fb",
"metadata": {},
"outputs": [],
"source": [
"from langsmith import Client\n",
"from langchain.smith import RunEvalConfig, run_on_dataset\n",
" return EvaluationResult(key=\"check_execution\", score=score)\n",
"\n",
"# Config\n",
"evaluation_config = RunEvalConfig(\n",
" custom_evaluators = [check_import,check_execution],\n",
")\n",
"\n",
"client = Client()\n",
"\n",
"config = {\"recursion_limit\": 50}\n",
"def model(input):\n",
" return app.invoke({\"keys\":{**input, \"docs\": concatenated_content}})\n",
" return app.invoke({\"keys\":{**input, \"docs\": concatenated_content, \"iterations\":0}},config=config)\n",
"\n",
"run_id = uuid.uuid4().hex[:4]\n",
"project_name = \"context-stuffing-with-langgraph\"\n",
"client.run_on_dataset(\n",
" dataset_name=\"lcel-teacher-eval\",\n",
" llm_or_chain_factory=model,\n",
" evaluation=evaluation_config,\n",
" project_name=f\"{run_id}-{project_name}\",\n",
")"
]
},