WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-08-22 23:52:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add Claude3 evals

Browse Source
This commit is contained in:
Lance Martin
2024-04-07 15:55:37 -07:00
parent 520443d6cd
commit 04668df9bc
1 changed files with 62 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/code_assistant/langgraph_code_assistant.ipynb
+62 -25
View File
@@ -47,7 +47,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 1,
"id": "c2eb35d1-4990-47dc-a5c4-208bae588a82",
"metadata": {},
"outputs": [],
@@ -92,7 +92,6 @@
"outputs": [],
"source": [
"from langchain_openai import ChatOpenAI\n",
"from langchain_anthropic import ChatAnthropic\n",
"from langchain_core.prompts import ChatPromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"\n",
@@ -126,14 +125,18 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 3,
"id": "cd30b67d-96db-4e51-a540-ae23fcc1f878",
"metadata": {},
"outputs": [],
"source": [
"from langchain_anthropic import ChatAnthropic\n",
"from langchain_core.prompts import ChatPromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"\n",
"### Anthropic\n",
"\n",
"# Important for getting tool use\n",
"# Prompt to enforce tool use\n",
"code_gen_prompt_claude = ChatPromptTemplate.from_messages(\n",
" [(\"system\",\"\"\"<instructions> You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is the LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user question based on the \\n \n",
@@ -142,8 +145,19 @@
" Invoke the code tool to structure the output correctly. </instructions> \\n Here is the user question:\"\"\",),\n",
" (\"placeholder\", \"{messages}\"),])\n",
"\n",
"# Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
" description = \"Schema for code solutions to questions about LCEL.\"\n",
"\n",
"\n",
"# LLM\n",
"expt_llm = \"claude-3-haiku-20240307\" # claude-3-opus-20240229\n",
"# expt_llm = \"claude-3-haiku-20240307\" \n",
"expt_llm = \"claude-3-opus-20240229\" \n",
"llm = ChatAnthropic(\n",
" model=expt_llm,\n",
" default_headers={\"anthropic-beta\": \"tools-2024-04-04\"},\n",
@@ -177,14 +191,14 @@
"code_chain_claude_raw = code_gen_prompt_claude | structured_llm_claude | check_claude_output\n",
"\n",
"def insert_errors(inputs):\n",
" \"\"\"Insert errors in the messages\"\"\"\n",
" \"\"\"Insert errors for tool parsing in the messages\"\"\"\n",
" \n",
" # Get errors\n",
" error = inputs[\"error\"]\n",
" messages = inputs[\"messages\"]\n",
" messages += [\n",
" (\n",
" \"user\",\n",
" \"assistant\",\n",
" f\"Retry. You are required to fix the parsing errors: {error} \\n\\n You must invoke the provided tool.\",\n",
" )\n",
" ]\n",
@@ -196,11 +210,34 @@
"# This will be run as a fallback chain\n",
"fallback_chain = insert_errors | code_chain_claude_raw\n",
"N = 3 # Max re-tries\n",
"code_gen_chain = code_chain_claude_raw.with_fallbacks(fallbacks=[fallback_chain] * N, exception_key=\"error\")\n",
"code_gen_chain_re_try = code_chain_claude_raw.with_fallbacks(fallbacks=[fallback_chain] * N, exception_key=\"error\")\n",
"\n",
"def parse_output(solution):\n",
" \"\"\"When we add 'include_raw=True' to structured output, \n",
" it will return a dict w 'raw', 'parsed', 'parsing_error'. \"\"\"\n",
" \n",
" return solution['parsed']\n",
"\n",
"# Wtih re-try\n",
"code_gen_chain = code_gen_chain_re_try | parse_output\n",
"\n",
"# No re-try\n",
"code_gen_chain = code_gen_prompt_claude | structured_llm_claude | parse_output"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9f14750f-dddc-485b-ba29-5392cdf4ba43",
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"# Test\n",
"question = \"How do I build a RAG chain in LCEL?\"\n",
"# solution = code_gen_chain_claude.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})"
"solution = code_gen_chain.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})\n",
"solution"
]
},
{
@@ -215,7 +252,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 4,
"id": "c185f1a2-e943-4bed-b833-4243c9c64092",
"metadata": {},
"outputs": [],
@@ -251,7 +288,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 5,
"id": "b70e8301-63ae-4f7e-ad8f-c9a052fe3566",
"metadata": {},
"outputs": [],
@@ -291,7 +328,7 @@
"\n",
" # We have been routed back to generation with an error\n",
" if error == \"yes\":\n",
" messages += [(\"user\",\"Now, try again. Be sure to structure your answer with a prefix, imports, and code block:\")]\n",
" messages += [(\"user\",\"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\")]\n",
" \n",
" # Solution\n",
" code_solution = code_gen_chain.invoke({\"context\": concatenated_content, \"messages\" : messages})\n",
@@ -329,7 +366,7 @@
" exec(imports)\n",
" except Exception as e:\n",
" print(\"---CODE IMPORT CHECK: FAILED---\")\n",
" error_message = [(\"assistant\", f\"Your solution failed the import test: {e}\")]\n",
" error_message = [(\"user\", f\"Your solution failed the import test: {e}\")]\n",
" messages += error_message\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"yes\"}\n",
" \n",
@@ -338,7 +375,7 @@
" exec(imports + \"\\n\" + code)\n",
" except Exception as e:\n",
" print(\"---CODE BLOCK CHECK: FAILED---\")\n",
" error_message = [(\"assistant\", f\"Your solution failed the code execution test: {e}\")]\n",
" error_message = [(\"user\", f\"Your solution failed the code execution test: {e}\")]\n",
" messages += error_message\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"yes\"}\n",
" \n",
@@ -402,7 +439,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 6,
"id": "f66b4e00-4731-42c8-bc38-72dd0ff7c92c",
"metadata": {},
"outputs": [],
@@ -465,7 +502,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 7,
"id": "678e8954-56b5-4cc6-be26-f7f2a060b242",
"metadata": {},
"outputs": [],
@@ -496,7 +533,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"id": "455a34ea-52cb-4ae5-9f4a-7e4a08cd0c09",
"metadata": {},
"outputs": [],
@@ -531,7 +568,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"id": "c8fa6bcb-b245-4422-b79a-582cd8a7d7ea",
"metadata": {},
"outputs": [],
@@ -551,7 +588,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"id": "d9c57468-97f6-47d6-a5e9-c09b53bfdd83",
"metadata": {},
"outputs": [],
@@ -598,7 +635,7 @@
" data=dataset_name,\n",
" evaluators=code_evalulator,\n",
" experiment_prefix=f\"test-with-langgraph-{expt_llm}-{flag}\",\n",
" max_concurrency=10,\n",
" max_concurrency=2,\n",
" metadata={\n",
" \"llm\": expt_llm,\n",
" \"feedback\": flag,\n",
@@ -611,11 +648,11 @@
"id": "d69da747-b4ea-455d-9314-60c3d9d30549",
"metadata": {},
"source": [
"Results:\n",
"`Results:`\n",
"\n",
"LangGraph w/o reflection performs the best by a wide margin.\n",
"\n",
"Reflection may confuse the re-try, and needs prompt engineering.\n",
"* `LangGraph outperforms base case`: adding re-try loop improve performance\n",
"* `Reflection did not help`: reflection prior to re-try regression vs just passing errors directly back to the LLM\n",
"* `GPT-4 outperforms Claude3`: Claude3 had 3 and 1 run fail due to tool-use error for Opus and Haiku, repspectively\n",
"\n",
"https://smith.langchain.com/public/78a3d858-c811-4e46-91cb-0f10ef56260b/d"
]
@@ -623,7 +660,7 @@
{
"cell_type": "code",
"execution_count": null,
"id": "96f85440-3622-4ae3-ae6f-7c0613466ffb",
"id": "a42333c3-c098-4576-ae2a-0258de64ece2",
"metadata": {},
"outputs": [],
"source": []