WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-08-23 08:02:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update notebooks to use bind_tools (#394)

Browse Source
This commit is contained in:
William FH
2024-05-04 01:30:52 -07:00
committed by GitHub
parent 21b8cbfd33
commit e0770d68b3
29 changed files with 1466 additions and 949 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/code_assistant/langgraph_code_assistant.ipynb
+138 -64
View File
@@ -34,7 +34,7 @@
"metadata": {},
"outputs": [],
"source": [
" ! pip install -U langchain_community langchain-openai langchain-anthropic langchain langgraph bs4"
"! pip install -U langchain_community langchain-openai langchain-anthropic langchain langgraph bs4"
]
},
{
@@ -97,16 +97,22 @@
"\n",
"### OpenAI\n",
"\n",
"# Grader prompt \n",
"# Grader prompt\n",
"code_gen_prompt = ChatPromptTemplate.from_messages(\n",
" [(\"system\",\"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" [\n",
" (\n",
" \"system\",\n",
" \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is a full set of LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user \n",
" question based on the above provided documentation. Ensure any code you provide can be executed \\n \n",
" with all required imports and variables defined. Structure your answer with a description of the code solution. \\n\n",
" Then list the imports. And finally list the functioning code block. Here is the user question:\"\"\"),\n",
" (\"placeholder\", \"{messages}\")]\n",
" Then list the imports. And finally list the functioning code block. Here is the user question:\"\"\",\n",
" ),\n",
" (\"placeholder\", \"{messages}\"),\n",
" ]\n",
")\n",
"\n",
"\n",
"# Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
@@ -116,6 +122,7 @@
" code: str = Field(description=\"Code block not including import statements\")\n",
" description = \"Schema for code solutions to questions about LCEL.\"\n",
"\n",
"\n",
"expt_llm = \"gpt-4-0125-preview\"\n",
"llm = ChatOpenAI(temperature=0, model=expt_llm)\n",
"code_gen_chain = code_gen_prompt | llm.with_structured_output(code)\n",
@@ -138,12 +145,19 @@
"\n",
"# Prompt to enforce tool use\n",
"code_gen_prompt_claude = ChatPromptTemplate.from_messages(\n",
" [(\"system\",\"\"\"<instructions> You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" [\n",
" (\n",
" \"system\",\n",
" \"\"\"<instructions> You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is the LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user question based on the \\n \n",
" above provided documentation. Ensure any code you provide can be executed with all required imports and variables \\n\n",
" defined. Structure your answer: 1) a prefix describing the code solution, 2) the imports, 3) the functioning code block. \\n\n",
" Invoke the code tool to structure the output correctly. </instructions> \\n Here is the user question:\"\"\",),\n",
" (\"placeholder\", \"{messages}\"),])\n",
" Invoke the code tool to structure the output correctly. </instructions> \\n Here is the user question:\"\"\",\n",
" ),\n",
" (\"placeholder\", \"{messages}\"),\n",
" ]\n",
")\n",
"\n",
"\n",
"# Data model\n",
"class code(BaseModel):\n",
@@ -156,8 +170,8 @@
"\n",
"\n",
"# LLM\n",
"# expt_llm = \"claude-3-haiku-20240307\" \n",
"expt_llm = \"claude-3-opus-20240229\" \n",
"# expt_llm = \"claude-3-haiku-20240307\"\n",
"expt_llm = \"claude-3-opus-20240229\"\n",
"llm = ChatAnthropic(\n",
" model=expt_llm,\n",
" default_headers={\"anthropic-beta\": \"tools-2024-04-04\"},\n",
@@ -165,6 +179,7 @@
"\n",
"structured_llm_claude = llm.with_structured_output(code, include_raw=True)\n",
"\n",
"\n",
"# Optional: Check for errors in case tool use is flaky\n",
"def check_claude_output(tool_output):\n",
" \"\"\"Check for parse error or failure to call the tool\"\"\"\n",
@@ -179,7 +194,7 @@
" f\"Error parsing your output! Be sure to invoke the tool. Output: {raw_output}. \\n Parse error: {error}\"\n",
" )\n",
"\n",
" # Tool was not invoked \n",
" # Tool was not invoked\n",
" elif not tool_output[\"parsed\"]:\n",
" print(\"Failed to invoke tool!\")\n",
" raise ValueError(\n",
@@ -187,12 +202,16 @@
" )\n",
" return tool_output\n",
"\n",
"\n",
"# Chain with output check\n",
"code_chain_claude_raw = code_gen_prompt_claude | structured_llm_claude | check_claude_output\n",
"code_chain_claude_raw = (\n",
" code_gen_prompt_claude | structured_llm_claude | check_claude_output\n",
")\n",
"\n",
"\n",
"def insert_errors(inputs):\n",
" \"\"\"Insert errors for tool parsing in the messages\"\"\"\n",
" \n",
"\n",
" # Get errors\n",
" error = inputs[\"error\"]\n",
" messages = inputs[\"messages\"]\n",
@@ -207,19 +226,24 @@
" \"context\": inputs[\"context\"],\n",
" }\n",
"\n",
"\n",
"# This will be run as a fallback chain\n",
"fallback_chain = insert_errors | code_chain_claude_raw\n",
"N = 3 # Max re-tries\n",
"code_gen_chain_re_try = code_chain_claude_raw.with_fallbacks(fallbacks=[fallback_chain] * N, exception_key=\"error\")\n",
"code_gen_chain_re_try = code_chain_claude_raw.with_fallbacks(\n",
" fallbacks=[fallback_chain] * N, exception_key=\"error\"\n",
")\n",
"\n",
"\n",
"def parse_output(solution):\n",
" \"\"\"When we add 'include_raw=True' to structured output, \n",
" it will return a dict w 'raw', 'parsed', 'parsing_error'. \"\"\"\n",
" \n",
" return solution['parsed']\n",
" \"\"\"When we add 'include_raw=True' to structured output,\n",
" it will return a dict w 'raw', 'parsed', 'parsing_error'.\"\"\"\n",
"\n",
" return solution[\"parsed\"]\n",
"\n",
"\n",
"# Wtih re-try to correct for failure to invoke tool\n",
"# TODO: Annoying errors w/ \"user\" vs \"assistant\" \n",
"# TODO: Annoying errors w/ \"user\" vs \"assistant\"\n",
"# Roles must alternate between \"user\" and \"assistant\", but found multiple \"user\" roles in a row\n",
"code_gen_chain = code_gen_chain_re_try | parse_output\n",
"\n",
@@ -238,7 +262,9 @@
"source": [
"# Test\n",
"question = \"How do I build a RAG chain in LCEL?\"\n",
"solution = code_gen_chain.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})\n",
"solution = code_gen_chain.invoke(\n",
" {\"context\": concatenated_content, \"messages\": [(\"user\", question)]}\n",
")\n",
"solution"
]
},
@@ -261,6 +287,7 @@
"source": [
"from typing import Dict, TypedDict, List\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
@@ -269,13 +296,13 @@
" error : Binary flag for control flow to indicate whether test error was tripped\n",
" messages : With user question, error messages, reasoning\n",
" generation : Code solution\n",
" iterations : Number of tries \n",
" iterations : Number of tries\n",
" \"\"\"\n",
"\n",
" error : str\n",
" messages : List\n",
" generation : str\n",
" iterations : int"
" error: str\n",
" messages: List\n",
" generation: str\n",
" iterations: int"
]
},
{
@@ -306,10 +333,11 @@
"max_iterations = 3\n",
"# Reflect\n",
"# flag = 'reflect'\n",
"flag = 'do not reflect'\n",
" \n",
"flag = \"do not reflect\"\n",
"\n",
"### Nodes\n",
"\n",
"\n",
"def generate(state: GraphState):\n",
" \"\"\"\n",
" Generate a code solution\n",
@@ -322,7 +350,7 @@
" \"\"\"\n",
"\n",
" print(\"---GENERATING CODE SOLUTION---\")\n",
" \n",
"\n",
" # State\n",
" messages = state[\"messages\"]\n",
" iterations = state[\"iterations\"]\n",
@@ -330,16 +358,29 @@
"\n",
" # We have been routed back to generation with an error\n",
" if error == \"yes\":\n",
" messages += [(\"user\",\"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\")]\n",
" \n",
" messages += [\n",
" (\n",
" \"user\",\n",
" \"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\",\n",
" )\n",
" ]\n",
"\n",
" # Solution\n",
" code_solution = code_gen_chain.invoke({\"context\": concatenated_content, \"messages\" : messages})\n",
" messages += [(\"assistant\",f\"{code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\")]\n",
" \n",
" code_solution = code_gen_chain.invoke(\n",
" {\"context\": concatenated_content, \"messages\": messages}\n",
" )\n",
" messages += [\n",
" (\n",
" \"assistant\",\n",
" f\"{code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n",
" )\n",
" ]\n",
"\n",
" # Increment\n",
" iterations = iterations + 1\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
"\n",
"\n",
"def code_check(state: GraphState):\n",
" \"\"\"\n",
" Check code\n",
@@ -352,7 +393,7 @@
" \"\"\"\n",
"\n",
" print(\"---CHECKING CODE---\")\n",
" \n",
"\n",
" # State\n",
" messages = state[\"messages\"]\n",
" code_solution = state[\"generation\"]\n",
@@ -370,8 +411,13 @@
" print(\"---CODE IMPORT CHECK: FAILED---\")\n",
" error_message = [(\"user\", f\"Your solution failed the import test: {e}\")]\n",
" messages += error_message\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"yes\"}\n",
" \n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"yes\",\n",
" }\n",
"\n",
" # Check execution\n",
" try:\n",
" exec(imports + \"\\n\" + code)\n",
@@ -379,11 +425,22 @@
" print(\"---CODE BLOCK CHECK: FAILED---\")\n",
" error_message = [(\"user\", f\"Your solution failed the code execution test: {e}\")]\n",
" messages += error_message\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"yes\"}\n",
" \n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"yes\",\n",
" }\n",
"\n",
" # No errors\n",
" print(\"---NO CODE TEST FAILURES---\")\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"no\"}\n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"no\",\n",
" }\n",
"\n",
"\n",
"def reflect(state: GraphState):\n",
" \"\"\"\n",
@@ -397,24 +454,33 @@
" \"\"\"\n",
"\n",
" print(\"---GENERATING CODE SOLUTION---\")\n",
" \n",
"\n",
" # State\n",
" messages = state[\"messages\"]\n",
" iterations = state[\"iterations\"]\n",
" code_solution = state[\"generation\"]\n",
"\n",
" # Prompt reflection\n",
" reflection_message = [(\"user\", \"\"\"You tried to solve this problem and failed a unit test. Reflect on this failure\n",
" reflection_message = [\n",
" (\n",
" \"user\",\n",
" \"\"\"You tried to solve this problem and failed a unit test. Reflect on this failure\n",
" given the provided documentation. Write a few key suggestions based on the \n",
" documentation to avoid making this mistake again.\"\"\")]\n",
" \n",
" documentation to avoid making this mistake again.\"\"\",\n",
" )\n",
" ]\n",
"\n",
" # Add reflection\n",
" reflections = code_gen_chain.invoke({\"context\" : concatenated_content, \"messages\" : messages})\n",
" messages += [(\"assistant\" , f\"Here are reflections on the error: {reflections}\")]\n",
" reflections = code_gen_chain.invoke(\n",
" {\"context\": concatenated_content, \"messages\": messages}\n",
" )\n",
" messages += [(\"assistant\", f\"Here are reflections on the error: {reflections}\")]\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
"\n",
"\n",
"### Edges\n",
"\n",
"\n",
"def decide_to_finish(state: GraphState):\n",
" \"\"\"\n",
" Determines whether to finish.\n",
@@ -433,7 +499,7 @@
" return \"end\"\n",
" else:\n",
" print(\"---DECISION: RE-TRY SOLUTION---\")\n",
" if flag == 'reflect':\n",
" if flag == \"reflect\":\n",
" return \"reflect\"\n",
" else:\n",
" return \"generate\""
@@ -479,7 +545,7 @@
"outputs": [],
"source": [
"question = \"How can I directly pass a string to a runnable and use it to construct the input needed for my prompt?\"\n",
"app.invoke({\"messages\":[(\"user\",question)],\"iterations\":0})"
"app.invoke({\"messages\": [(\"user\", question)], \"iterations\": 0})"
]
},
{
@@ -510,6 +576,7 @@
"outputs": [],
"source": [
"import langsmith\n",
"\n",
"client = langsmith.Client()"
]
},
@@ -521,7 +588,9 @@
"outputs": [],
"source": [
"# Clone the dataset to your tenant to use it\n",
"public_dataset = (\"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\")\n",
"public_dataset = (\n",
" \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n",
")\n",
"client.clone_public_dataset(public_dataset)"
]
},
@@ -542,22 +611,24 @@
"source": [
"from langsmith.schemas import Example, Run\n",
"\n",
"def check_import(run: Run, example: Example) -> dict: \n",
"\n",
"def check_import(run: Run, example: Example) -> dict:\n",
" imports = run.outputs.get(\"imports\")\n",
" try:\n",
" exec(imports)\n",
" return {\"key\": \"import_check\" , \"score\": 1} \n",
" return {\"key\": \"import_check\", \"score\": 1}\n",
" except:\n",
" return {\"key\": \"import_check\" , \"score\": 0} \n",
" return {\"key\": \"import_check\", \"score\": 0}\n",
"\n",
"def check_execution(run: Run, example: Example) -> dict: \n",
"\n",
"def check_execution(run: Run, example: Example) -> dict:\n",
" imports = run.outputs.get(\"imports\")\n",
" code = run.outputs.get(\"code\")\n",
" try:\n",
" exec(imports + \"\\n\" + code)\n",
" return {\"key\": \"code_execution_check\" , \"score\": 1} \n",
" return {\"key\": \"code_execution_check\", \"score\": 1}\n",
" except:\n",
" return {\"key\": \"code_execution_check\" , \"score\": 0} "
" return {\"key\": \"code_execution_check\", \"score\": 0}"
]
},
{
@@ -576,14 +647,17 @@
"outputs": [],
"source": [
"def predict_base_case(example: dict):\n",
" \"\"\" Context stuffing \"\"\"\n",
" solution = code_gen_chain.invoke({\"context\" : concatenated_content, \"messages\" : [(\"user\",example[\"question\"])]})\n",
" solution_structured = structured_code_formatter.invoke([(\"code\",solution)])\n",
" \"\"\"Context stuffing\"\"\"\n",
" solution = code_gen_chain.invoke(\n",
" {\"context\": concatenated_content, \"messages\": [(\"user\", example[\"question\"])]}\n",
" )\n",
" solution_structured = structured_code_formatter.invoke([(\"code\", solution)])\n",
" return {\"imports\": solution_structured.imports, \"code\": solution_structured.code}\n",
"\n",
"\n",
"def predict_langgraph(example: dict):\n",
" \"\"\" LangGraph \"\"\"\n",
" graph = app.invoke({\"messages\":[(\"user\",example[\"question\"])],\"iterations\":0})\n",
" \"\"\"LangGraph\"\"\"\n",
" graph = app.invoke({\"messages\": [(\"user\", example[\"question\"])], \"iterations\": 0})\n",
" solution = graph[\"generation\"]\n",
" return {\"imports\": solution.imports, \"code\": solution.code}"
]
@@ -598,7 +672,7 @@
"from langsmith.evaluation import evaluate\n",
"\n",
"# Evaluator\n",
"code_evalulator = [check_import,check_execution]\n",
"code_evalulator = [check_import, check_execution]\n",
"\n",
"# Dataset\n",
"dataset_name = \"test-LCEL-code-gen\""
@@ -616,10 +690,10 @@
" predict_base_case,\n",
" data=dataset_name,\n",
" evaluators=code_evalulator,\n",
" experiment_prefix=f\"test-without-langgraph-{expt_llm}\", \n",
" experiment_prefix=f\"test-without-langgraph-{expt_llm}\",\n",
" max_concurrency=2,\n",
" metadata={\n",
" \"llm\": expt_llm,\n",
" \"llm\": expt_llm,\n",
" },\n",
")"
]
@@ -639,8 +713,8 @@
" experiment_prefix=f\"test-with-langgraph-{expt_llm}-{flag}\",\n",
" max_concurrency=2,\n",
" metadata={\n",
" \"llm\": expt_llm,\n",
" \"feedback\": flag,\n",
" \"llm\": expt_llm,\n",
" \"feedback\": flag,\n",
" },\n",
")"
]