[Docs] Update notebooks to use START (#902)

This commit is contained in:
William FH
2024-07-01 21:36:34 -07:00
committed by GitHub
parent 267f5e5234
commit 727e63c01e
67 changed files with 1059 additions and 20258 deletions
@@ -34,9 +34,7 @@
"id": "e3900420",
"metadata": {},
"outputs": [],
"source": [
"! pip install -U langchain_community langchain-openai langchain-anthropic langchain langgraph bs4"
]
"source": ["! pip install -U langchain_community langchain-openai langchain-anthropic langchain langgraph bs4"]
},
{
"cell_type": "markdown",
@@ -54,24 +52,7 @@
"id": "c2eb35d1-4990-47dc-a5c4-208bae588a82",
"metadata": {},
"outputs": [],
"source": [
"from bs4 import BeautifulSoup as Soup\n",
"from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n",
"\n",
"# LCEL docs\n",
"url = \"https://python.langchain.com/v0.2/docs/concepts/#langchain-expression-language-lcel\"\n",
"loader = RecursiveUrlLoader(\n",
" url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n",
")\n",
"docs = loader.load()\n",
"\n",
"# Sort the list based on the URLs and get the text\n",
"d_sorted = sorted(docs, key=lambda x: x.metadata[\"source\"])\n",
"d_reversed = list(reversed(d_sorted))\n",
"concatenated_content = \"\\n\\n\\n --- \\n\\n\\n\".join(\n",
" [doc.page_content for doc in d_reversed]\n",
")"
]
"source": ["from bs4 import BeautifulSoup as Soup\nfrom langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n\n# LCEL docs\nurl = \"https://python.langchain.com/v0.2/docs/concepts/#langchain-expression-language-lcel\"\nloader = RecursiveUrlLoader(\n url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n)\ndocs = loader.load()\n\n# Sort the list based on the URLs and get the text\nd_sorted = sorted(docs, key=lambda x: x.metadata[\"source\"])\nd_reversed = list(reversed(d_sorted))\nconcatenated_content = \"\\n\\n\\n --- \\n\\n\\n\".join(\n [doc.page_content for doc in d_reversed]\n)"]
},
{
"cell_type": "markdown",
@@ -93,45 +74,7 @@
"id": "3ba3df70-f6b4-4ea5-a210-e10944960bc6",
"metadata": {},
"outputs": [],
"source": [
"from langchain_core.prompts import ChatPromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"### OpenAI\n",
"\n",
"# Grader prompt\n",
"code_gen_prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\n",
" \"system\",\n",
" \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is a full set of LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user \n",
" question based on the above provided documentation. Ensure any code you provide can be executed \\n \n",
" with all required imports and variables defined. Structure your answer with a description of the code solution. \\n\n",
" Then list the imports. And finally list the functioning code block. Here is the user question:\"\"\",\n",
" ),\n",
" (\"placeholder\", \"{messages}\"),\n",
" ]\n",
")\n",
"\n",
"\n",
"# Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
" description = \"Schema for code solutions to questions about LCEL.\"\n",
"\n",
"\n",
"expt_llm = \"gpt-4-0125-preview\"\n",
"llm = ChatOpenAI(temperature=0, model=expt_llm)\n",
"code_gen_chain = code_gen_prompt | llm.with_structured_output(code)\n",
"question = \"How do I build a RAG chain in LCEL?\"\n",
"# solution = code_gen_chain_oai.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})"
]
"source": ["from langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\n### OpenAI\n\n# Grader prompt\ncode_gen_prompt = ChatPromptTemplate.from_messages(\n [\n (\n \"system\",\n \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n Here is a full set of LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user \n question based on the above provided documentation. Ensure any code you provide can be executed \\n \n with all required imports and variables defined. Structure your answer with a description of the code solution. \\n\n Then list the imports. And finally list the functioning code block. Here is the user question:\"\"\",\n ),\n (\"placeholder\", \"{messages}\"),\n ]\n)\n\n\n# Data model\nclass code(BaseModel):\n \"\"\"Code output\"\"\"\n\n prefix: str = Field(description=\"Description of the problem and approach\")\n imports: str = Field(description=\"Code block import statements\")\n code: str = Field(description=\"Code block not including import statements\")\n description = \"Schema for code solutions to questions about LCEL.\"\n\n\nexpt_llm = \"gpt-4-0125-preview\"\nllm = ChatOpenAI(temperature=0, model=expt_llm)\ncode_gen_chain = code_gen_prompt | llm.with_structured_output(code)\nquestion = \"How do I build a RAG chain in LCEL?\"\n# solution = code_gen_chain_oai.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})"]
},
{
"cell_type": "code",
@@ -139,118 +82,7 @@
"id": "cd30b67d-96db-4e51-a540-ae23fcc1f878",
"metadata": {},
"outputs": [],
"source": [
"from langchain_anthropic import ChatAnthropic\n",
"from langchain_core.prompts import ChatPromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"\n",
"### Anthropic\n",
"\n",
"# Prompt to enforce tool use\n",
"code_gen_prompt_claude = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\n",
" \"system\",\n",
" \"\"\"<instructions> You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is the LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user question based on the \\n \n",
" above provided documentation. Ensure any code you provide can be executed with all required imports and variables \\n\n",
" defined. Structure your answer: 1) a prefix describing the code solution, 2) the imports, 3) the functioning code block. \\n\n",
" Invoke the code tool to structure the output correctly. </instructions> \\n Here is the user question:\"\"\",\n",
" ),\n",
" (\"placeholder\", \"{messages}\"),\n",
" ]\n",
")\n",
"\n",
"\n",
"# Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
" description = \"Schema for code solutions to questions about LCEL.\"\n",
"\n",
"\n",
"# LLM\n",
"# expt_llm = \"claude-3-haiku-20240307\"\n",
"expt_llm = \"claude-3-opus-20240229\"\n",
"llm = ChatAnthropic(\n",
" model=expt_llm,\n",
" default_headers={\"anthropic-beta\": \"tools-2024-04-04\"},\n",
")\n",
"\n",
"structured_llm_claude = llm.with_structured_output(code, include_raw=True)\n",
"\n",
"\n",
"# Optional: Check for errors in case tool use is flaky\n",
"def check_claude_output(tool_output):\n",
" \"\"\"Check for parse error or failure to call the tool\"\"\"\n",
"\n",
" # Error with parsing\n",
" if tool_output[\"parsing_error\"]:\n",
" # Report back output and parsing errors\n",
" print(\"Parsing error!\")\n",
" raw_output = str(tool_output[\"raw\"].content)\n",
" error = tool_output[\"parsing_error\"]\n",
" raise ValueError(\n",
" f\"Error parsing your output! Be sure to invoke the tool. Output: {raw_output}. \\n Parse error: {error}\"\n",
" )\n",
"\n",
" # Tool was not invoked\n",
" elif not tool_output[\"parsed\"]:\n",
" print(\"Failed to invoke tool!\")\n",
" raise ValueError(\n",
" \"You did not use the provided tool! Be sure to invoke the tool to structure the output.\"\n",
" )\n",
" return tool_output\n",
"\n",
"\n",
"# Chain with output check\n",
"code_chain_claude_raw = (\n",
" code_gen_prompt_claude | structured_llm_claude | check_claude_output\n",
")\n",
"\n",
"\n",
"def insert_errors(inputs):\n",
" \"\"\"Insert errors for tool parsing in the messages\"\"\"\n",
"\n",
" # Get errors\n",
" error = inputs[\"error\"]\n",
" messages = inputs[\"messages\"]\n",
" messages += [\n",
" (\n",
" \"assistant\",\n",
" f\"Retry. You are required to fix the parsing errors: {error} \\n\\n You must invoke the provided tool.\",\n",
" )\n",
" ]\n",
" return {\n",
" \"messages\": messages,\n",
" \"context\": inputs[\"context\"],\n",
" }\n",
"\n",
"\n",
"# This will be run as a fallback chain\n",
"fallback_chain = insert_errors | code_chain_claude_raw\n",
"N = 3 # Max re-tries\n",
"code_gen_chain_re_try = code_chain_claude_raw.with_fallbacks(\n",
" fallbacks=[fallback_chain] * N, exception_key=\"error\"\n",
")\n",
"\n",
"\n",
"def parse_output(solution):\n",
" \"\"\"When we add 'include_raw=True' to structured output,\n",
" it will return a dict w 'raw', 'parsed', 'parsing_error'.\"\"\"\n",
"\n",
" return solution[\"parsed\"]\n",
"\n",
"\n",
"# Optional: With re-try to correct for failure to invoke tool\n",
"code_gen_chain = code_gen_chain_re_try | parse_output\n",
"\n",
"# No re-try\n",
"code_gen_chain = code_gen_prompt_claude | structured_llm_claude | parse_output"
]
"source": ["from langchain_anthropic import ChatAnthropic\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\n\n### Anthropic\n\n# Prompt to enforce tool use\ncode_gen_prompt_claude = ChatPromptTemplate.from_messages(\n [\n (\n \"system\",\n \"\"\"<instructions> You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n Here is the LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user question based on the \\n \n above provided documentation. Ensure any code you provide can be executed with all required imports and variables \\n\n defined. Structure your answer: 1) a prefix describing the code solution, 2) the imports, 3) the functioning code block. \\n\n Invoke the code tool to structure the output correctly. </instructions> \\n Here is the user question:\"\"\",\n ),\n (\"placeholder\", \"{messages}\"),\n ]\n)\n\n\n# Data model\nclass code(BaseModel):\n \"\"\"Code output\"\"\"\n\n prefix: str = Field(description=\"Description of the problem and approach\")\n imports: str = Field(description=\"Code block import statements\")\n code: str = Field(description=\"Code block not including import statements\")\n description = \"Schema for code solutions to questions about LCEL.\"\n\n\n# LLM\n# expt_llm = \"claude-3-haiku-20240307\"\nexpt_llm = \"claude-3-opus-20240229\"\nllm = ChatAnthropic(\n model=expt_llm,\n default_headers={\"anthropic-beta\": \"tools-2024-04-04\"},\n)\n\nstructured_llm_claude = llm.with_structured_output(code, include_raw=True)\n\n\n# Optional: Check for errors in case tool use is flaky\ndef check_claude_output(tool_output):\n \"\"\"Check for parse error or failure to call the tool\"\"\"\n\n # Error with parsing\n if tool_output[\"parsing_error\"]:\n # Report back output and parsing errors\n print(\"Parsing error!\")\n raw_output = str(tool_output[\"raw\"].content)\n error = tool_output[\"parsing_error\"]\n raise ValueError(\n f\"Error parsing your output! Be sure to invoke the tool. Output: {raw_output}. \\n Parse error: {error}\"\n )\n\n # Tool was not invoked\n elif not tool_output[\"parsed\"]:\n print(\"Failed to invoke tool!\")\n raise ValueError(\n \"You did not use the provided tool! Be sure to invoke the tool to structure the output.\"\n )\n return tool_output\n\n\n# Chain with output check\ncode_chain_claude_raw = (\n code_gen_prompt_claude | structured_llm_claude | check_claude_output\n)\n\n\ndef insert_errors(inputs):\n \"\"\"Insert errors for tool parsing in the messages\"\"\"\n\n # Get errors\n error = inputs[\"error\"]\n messages = inputs[\"messages\"]\n messages += [\n (\n \"assistant\",\n f\"Retry. You are required to fix the parsing errors: {error} \\n\\n You must invoke the provided tool.\",\n )\n ]\n return {\n \"messages\": messages,\n \"context\": inputs[\"context\"],\n }\n\n\n# This will be run as a fallback chain\nfallback_chain = insert_errors | code_chain_claude_raw\nN = 3 # Max re-tries\ncode_gen_chain_re_try = code_chain_claude_raw.with_fallbacks(\n fallbacks=[fallback_chain] * N, exception_key=\"error\"\n)\n\n\ndef parse_output(solution):\n \"\"\"When we add 'include_raw=True' to structured output,\n it will return a dict w 'raw', 'parsed', 'parsing_error'.\"\"\"\n\n return solution[\"parsed\"]\n\n\n# Optional: With re-try to correct for failure to invoke tool\ncode_gen_chain = code_gen_chain_re_try | parse_output\n\n# No re-try\ncode_gen_chain = code_gen_prompt_claude | structured_llm_claude | parse_output"]
},
{
"cell_type": "code",
@@ -260,14 +92,7 @@
"scrolled": true
},
"outputs": [],
"source": [
"# Test\n",
"question = \"How do I build a RAG chain in LCEL?\"\n",
"solution = code_gen_chain.invoke(\n",
" {\"context\": concatenated_content, \"messages\": [(\"user\", question)]}\n",
")\n",
"solution"
]
"source": ["# Test\nquestion = \"How do I build a RAG chain in LCEL?\"\nsolution = code_gen_chain.invoke(\n {\"context\": concatenated_content, \"messages\": [(\"user\", question)]}\n)\nsolution"]
},
{
"cell_type": "markdown",
@@ -285,26 +110,7 @@
"id": "c185f1a2-e943-4bed-b833-4243c9c64092",
"metadata": {},
"outputs": [],
"source": [
"from typing import List, TypedDict\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
"\n",
" Attributes:\n",
" error : Binary flag for control flow to indicate whether test error was tripped\n",
" messages : With user question, error messages, reasoning\n",
" generation : Code solution\n",
" iterations : Number of tries\n",
" \"\"\"\n",
"\n",
" error: str\n",
" messages: List\n",
" generation: str\n",
" iterations: int"
]
"source": ["from typing import List, TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n error : Binary flag for control flow to indicate whether test error was tripped\n messages : With user question, error messages, reasoning\n generation : Code solution\n iterations : Number of tries\n \"\"\"\n\n error: str\n messages: List\n generation: str\n iterations: int"]
},
{
"cell_type": "markdown",
@@ -322,177 +128,7 @@
"id": "b70e8301-63ae-4f7e-ad8f-c9a052fe3566",
"metadata": {},
"outputs": [],
"source": [
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"\n",
"### Parameter\n",
"\n",
"# Max tries\n",
"max_iterations = 3\n",
"# Reflect\n",
"# flag = 'reflect'\n",
"flag = \"do not reflect\"\n",
"\n",
"### Nodes\n",
"\n",
"\n",
"def generate(state: GraphState):\n",
" \"\"\"\n",
" Generate a code solution\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, generation\n",
" \"\"\"\n",
"\n",
" print(\"---GENERATING CODE SOLUTION---\")\n",
"\n",
" # State\n",
" messages = state[\"messages\"]\n",
" iterations = state[\"iterations\"]\n",
" error = state[\"error\"]\n",
"\n",
" # We have been routed back to generation with an error\n",
" if error == \"yes\":\n",
" messages += [\n",
" (\n",
" \"user\",\n",
" \"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\",\n",
" )\n",
" ]\n",
"\n",
" # Solution\n",
" code_solution = code_gen_chain.invoke(\n",
" {\"context\": concatenated_content, \"messages\": messages}\n",
" )\n",
" messages += [\n",
" (\n",
" \"assistant\",\n",
" f\"{code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n",
" )\n",
" ]\n",
"\n",
" # Increment\n",
" iterations = iterations + 1\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
"\n",
"\n",
"def code_check(state: GraphState):\n",
" \"\"\"\n",
" Check code\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, error\n",
" \"\"\"\n",
"\n",
" print(\"---CHECKING CODE---\")\n",
"\n",
" # State\n",
" messages = state[\"messages\"]\n",
" code_solution = state[\"generation\"]\n",
" iterations = state[\"iterations\"]\n",
"\n",
" # Get solution components\n",
" imports = code_solution.imports\n",
" code = code_solution.code\n",
"\n",
" # Check imports\n",
" try:\n",
" exec(imports)\n",
" except Exception as e:\n",
" print(\"---CODE IMPORT CHECK: FAILED---\")\n",
" error_message = [(\"user\", f\"Your solution failed the import test: {e}\")]\n",
" messages += error_message\n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"yes\",\n",
" }\n",
"\n",
" # Check execution\n",
" try:\n",
" exec(imports + \"\\n\" + code)\n",
" except Exception as e:\n",
" print(\"---CODE BLOCK CHECK: FAILED---\")\n",
" error_message = [(\"user\", f\"Your solution failed the code execution test: {e}\")]\n",
" messages += error_message\n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"yes\",\n",
" }\n",
"\n",
" # No errors\n",
" print(\"---NO CODE TEST FAILURES---\")\n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"no\",\n",
" }\n",
"\n",
"\n",
"def reflect(state: GraphState):\n",
" \"\"\"\n",
" Reflect on errors\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, generation\n",
" \"\"\"\n",
"\n",
" print(\"---GENERATING CODE SOLUTION---\")\n",
"\n",
" # State\n",
" messages = state[\"messages\"]\n",
" iterations = state[\"iterations\"]\n",
" code_solution = state[\"generation\"]\n",
"\n",
" # Prompt reflection\n",
"\n",
" # Add reflection\n",
" reflections = code_gen_chain.invoke(\n",
" {\"context\": concatenated_content, \"messages\": messages}\n",
" )\n",
" messages += [(\"assistant\", f\"Here are reflections on the error: {reflections}\")]\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
"\n",
"\n",
"### Edges\n",
"\n",
"\n",
"def decide_to_finish(state: GraphState):\n",
" \"\"\"\n",
" Determines whether to finish.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Next node to call\n",
" \"\"\"\n",
" error = state[\"error\"]\n",
" iterations = state[\"iterations\"]\n",
"\n",
" if error == \"no\" or iterations == max_iterations:\n",
" print(\"---DECISION: FINISH---\")\n",
" return \"end\"\n",
" else:\n",
" print(\"---DECISION: RE-TRY SOLUTION---\")\n",
" if flag == \"reflect\":\n",
" return \"reflect\"\n",
" else:\n",
" return \"generate\""
]
"source": ["from langchain_core.pydantic_v1 import BaseModel, Field\n\n### Parameter\n\n# Max tries\nmax_iterations = 3\n# Reflect\n# flag = 'reflect'\nflag = \"do not reflect\"\n\n### Nodes\n\n\ndef generate(state: GraphState):\n \"\"\"\n Generate a code solution\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation\n \"\"\"\n\n print(\"---GENERATING CODE SOLUTION---\")\n\n # State\n messages = state[\"messages\"]\n iterations = state[\"iterations\"]\n error = state[\"error\"]\n\n # We have been routed back to generation with an error\n if error == \"yes\":\n messages += [\n (\n \"user\",\n \"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\",\n )\n ]\n\n # Solution\n code_solution = code_gen_chain.invoke(\n {\"context\": concatenated_content, \"messages\": messages}\n )\n messages += [\n (\n \"assistant\",\n f\"{code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n )\n ]\n\n # Increment\n iterations = iterations + 1\n return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n\n\ndef code_check(state: GraphState):\n \"\"\"\n Check code\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, error\n \"\"\"\n\n print(\"---CHECKING CODE---\")\n\n # State\n messages = state[\"messages\"]\n code_solution = state[\"generation\"]\n iterations = state[\"iterations\"]\n\n # Get solution components\n imports = code_solution.imports\n code = code_solution.code\n\n # Check imports\n try:\n exec(imports)\n except Exception as e:\n print(\"---CODE IMPORT CHECK: FAILED---\")\n error_message = [(\"user\", f\"Your solution failed the import test: {e}\")]\n messages += error_message\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"yes\",\n }\n\n # Check execution\n try:\n exec(imports + \"\\n\" + code)\n except Exception as e:\n print(\"---CODE BLOCK CHECK: FAILED---\")\n error_message = [(\"user\", f\"Your solution failed the code execution test: {e}\")]\n messages += error_message\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"yes\",\n }\n\n # No errors\n print(\"---NO CODE TEST FAILURES---\")\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"no\",\n }\n\n\ndef reflect(state: GraphState):\n \"\"\"\n Reflect on errors\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation\n \"\"\"\n\n print(\"---GENERATING CODE SOLUTION---\")\n\n # State\n messages = state[\"messages\"]\n iterations = state[\"iterations\"]\n code_solution = state[\"generation\"]\n\n # Prompt reflection\n\n # Add reflection\n reflections = code_gen_chain.invoke(\n {\"context\": concatenated_content, \"messages\": messages}\n )\n messages += [(\"assistant\", f\"Here are reflections on the error: {reflections}\")]\n return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n\n\n### Edges\n\n\ndef decide_to_finish(state: GraphState):\n \"\"\"\n Determines whether to finish.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Next node to call\n \"\"\"\n error = state[\"error\"]\n iterations = state[\"iterations\"]\n\n if error == \"no\" or iterations == max_iterations:\n print(\"---DECISION: FINISH---\")\n return \"end\"\n else:\n print(\"---DECISION: RE-TRY SOLUTION---\")\n if flag == \"reflect\":\n return \"reflect\"\n else:\n return \"generate\""]
},
{
"cell_type": "code",
@@ -500,31 +136,7 @@
"id": "f66b4e00-4731-42c8-bc38-72dd0ff7c92c",
"metadata": {},
"outputs": [],
"source": [
"from langgraph.graph import END, StateGraph\n",
"\n",
"workflow = StateGraph(GraphState)\n",
"\n",
"# Define the nodes\n",
"workflow.add_node(\"generate\", generate) # generation solution\n",
"workflow.add_node(\"check_code\", code_check) # check code\n",
"workflow.add_node(\"reflect\", reflect) # reflect\n",
"\n",
"# Build graph\n",
"workflow.set_entry_point(\"generate\")\n",
"workflow.add_edge(\"generate\", \"check_code\")\n",
"workflow.add_conditional_edges(\n",
" \"check_code\",\n",
" decide_to_finish,\n",
" {\n",
" \"end\": END,\n",
" \"reflect\": \"reflect\",\n",
" \"generate\": \"generate\",\n",
" },\n",
")\n",
"workflow.add_edge(\"reflect\", \"generate\")\n",
"app = workflow.compile()"
]
"source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"generate\", generate) # generation solution\nworkflow.add_node(\"check_code\", code_check) # check code\nworkflow.add_node(\"reflect\", reflect) # reflect\n\n# Build graph\nworkflow.add_edge(START, \"generate\")\nworkflow.add_edge(\"generate\", \"check_code\")\nworkflow.add_conditional_edges(\n \"check_code\",\n decide_to_finish,\n {\n \"end\": END,\n \"reflect\": \"reflect\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"reflect\", \"generate\")\napp = workflow.compile()"]
},
{
"cell_type": "code",
@@ -532,10 +144,7 @@
"id": "9bcaafe4-ddcf-4fab-8620-2d9b6c508f98",
"metadata": {},
"outputs": [],
"source": [
"question = \"How can I directly pass a string to a runnable and use it to construct the input needed for my prompt?\"\n",
"app.invoke({\"messages\": [(\"user\", question)], \"iterations\": 0})"
]
"source": ["question = \"How can I directly pass a string to a runnable and use it to construct the input needed for my prompt?\"\napp.invoke({\"messages\": [(\"user\", question)], \"iterations\": 0})"]
},
{
"cell_type": "markdown",
@@ -563,11 +172,7 @@
"id": "678e8954-56b5-4cc6-be26-f7f2a060b242",
"metadata": {},
"outputs": [],
"source": [
"import langsmith\n",
"\n",
"client = langsmith.Client()"
]
"source": ["import langsmith\n\nclient = langsmith.Client()"]
},
{
"cell_type": "code",
@@ -575,13 +180,7 @@
"id": "ef7cf662-7a6f-4dee-965c-6309d4045feb",
"metadata": {},
"outputs": [],
"source": [
"# Clone the dataset to your tenant to use it\n",
"public_dataset = (\n",
" \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n",
")\n",
"client.clone_public_dataset(public_dataset)"
]
"source": ["# Clone the dataset to your tenant to use it\npublic_dataset = (\n \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n)\nclient.clone_public_dataset(public_dataset)"]
},
{
"cell_type": "markdown",
@@ -597,28 +196,7 @@
"id": "455a34ea-52cb-4ae5-9f4a-7e4a08cd0c09",
"metadata": {},
"outputs": [],
"source": [
"from langsmith.schemas import Example, Run\n",
"\n",
"\n",
"def check_import(run: Run, example: Example) -> dict:\n",
" imports = run.outputs.get(\"imports\")\n",
" try:\n",
" exec(imports)\n",
" return {\"key\": \"import_check\", \"score\": 1}\n",
" except Exception:\n",
" return {\"key\": \"import_check\", \"score\": 0}\n",
"\n",
"\n",
"def check_execution(run: Run, example: Example) -> dict:\n",
" imports = run.outputs.get(\"imports\")\n",
" code = run.outputs.get(\"code\")\n",
" try:\n",
" exec(imports + \"\\n\" + code)\n",
" return {\"key\": \"code_execution_check\", \"score\": 1}\n",
" except Exception:\n",
" return {\"key\": \"code_execution_check\", \"score\": 0}"
]
"source": ["from langsmith.schemas import Example, Run\n\n\ndef check_import(run: Run, example: Example) -> dict:\n imports = run.outputs.get(\"imports\")\n try:\n exec(imports)\n return {\"key\": \"import_check\", \"score\": 1}\n except Exception:\n return {\"key\": \"import_check\", \"score\": 0}\n\n\ndef check_execution(run: Run, example: Example) -> dict:\n imports = run.outputs.get(\"imports\")\n code = run.outputs.get(\"code\")\n try:\n exec(imports + \"\\n\" + code)\n return {\"key\": \"code_execution_check\", \"score\": 1}\n except Exception:\n return {\"key\": \"code_execution_check\", \"score\": 0}"]
},
{
"cell_type": "markdown",
@@ -634,22 +212,7 @@
"id": "c8fa6bcb-b245-4422-b79a-582cd8a7d7ea",
"metadata": {},
"outputs": [],
"source": [
"def predict_base_case(example: dict):\n",
" \"\"\"Context stuffing\"\"\"\n",
" solution = code_gen_chain.invoke(\n",
" {\"context\": concatenated_content, \"messages\": [(\"user\", example[\"question\"])]}\n",
" )\n",
" solution_structured = code_gen_chain.invoke([(\"code\", solution)])\n",
" return {\"imports\": solution_structured.imports, \"code\": solution_structured.code}\n",
"\n",
"\n",
"def predict_langgraph(example: dict):\n",
" \"\"\"LangGraph\"\"\"\n",
" graph = app.invoke({\"messages\": [(\"user\", example[\"question\"])], \"iterations\": 0})\n",
" solution = graph[\"generation\"]\n",
" return {\"imports\": solution.imports, \"code\": solution.code}"
]
"source": ["def predict_base_case(example: dict):\n \"\"\"Context stuffing\"\"\"\n solution = code_gen_chain.invoke(\n {\"context\": concatenated_content, \"messages\": [(\"user\", example[\"question\"])]}\n )\n solution_structured = code_gen_chain.invoke([(\"code\", solution)])\n return {\"imports\": solution_structured.imports, \"code\": solution_structured.code}\n\n\ndef predict_langgraph(example: dict):\n \"\"\"LangGraph\"\"\"\n graph = app.invoke({\"messages\": [(\"user\", example[\"question\"])], \"iterations\": 0})\n solution = graph[\"generation\"]\n return {\"imports\": solution.imports, \"code\": solution.code}"]
},
{
"cell_type": "code",
@@ -657,15 +220,7 @@
"id": "d9c57468-97f6-47d6-a5e9-c09b53bfdd83",
"metadata": {},
"outputs": [],
"source": [
"from langsmith.evaluation import evaluate\n",
"\n",
"# Evaluator\n",
"code_evalulator = [check_import, check_execution]\n",
"\n",
"# Dataset\n",
"dataset_name = \"test-LCEL-code-gen\""
]
"source": ["from langsmith.evaluation import evaluate\n\n# Evaluator\ncode_evalulator = [check_import, check_execution]\n\n# Dataset\ndataset_name = \"test-LCEL-code-gen\""]
},
{
"cell_type": "code",
@@ -673,19 +228,7 @@
"id": "2dacccf0-d73f-4017-aaf0-9806ffe5bd2c",
"metadata": {},
"outputs": [],
"source": [
"# Run base case\n",
"experiment_results_ = evaluate(\n",
" predict_base_case,\n",
" data=dataset_name,\n",
" evaluators=code_evalulator,\n",
" experiment_prefix=f\"test-without-langgraph-{expt_llm}\",\n",
" max_concurrency=2,\n",
" metadata={\n",
" \"llm\": expt_llm,\n",
" },\n",
")"
]
"source": ["# Run base case\nexperiment_results_ = evaluate(\n predict_base_case,\n data=dataset_name,\n evaluators=code_evalulator,\n experiment_prefix=f\"test-without-langgraph-{expt_llm}\",\n max_concurrency=2,\n metadata={\n \"llm\": expt_llm,\n },\n)"]
},
{
"cell_type": "code",
@@ -693,20 +236,7 @@
"id": "71d90f9e-9dad-410c-a709-093d275029ae",
"metadata": {},
"outputs": [],
"source": [
"# Run with langgraph\n",
"experiment_results = evaluate(\n",
" predict_langgraph,\n",
" data=dataset_name,\n",
" evaluators=code_evalulator,\n",
" experiment_prefix=f\"test-with-langgraph-{expt_llm}-{flag}\",\n",
" max_concurrency=2,\n",
" metadata={\n",
" \"llm\": expt_llm,\n",
" \"feedback\": flag,\n",
" },\n",
")"
]
"source": ["# Run with langgraph\nexperiment_results = evaluate(\n predict_langgraph,\n data=dataset_name,\n evaluators=code_evalulator,\n experiment_prefix=f\"test-with-langgraph-{expt_llm}-{flag}\",\n max_concurrency=2,\n metadata={\n \"llm\": expt_llm,\n \"feedback\": flag,\n },\n)"]
},
{
"cell_type": "markdown",
@@ -728,7 +258,7 @@
"id": "a42333c3-c098-4576-ae2a-0258de64ece2",
"metadata": {},
"outputs": [],
"source": []
"source": [""]
}
],
"metadata": {
@@ -33,9 +33,7 @@
"id": "e501686f-323f-4b87-8f9c-8ba89133078b",
"metadata": {},
"outputs": [],
"source": [
"! pip install -U langchain_community langchain-mistralai langchain langgraph"
]
"source": ["! pip install -U langchain_community langchain-mistralai langchain langgraph"]
},
{
"cell_type": "markdown",
@@ -53,12 +51,7 @@
"id": "982e4609-86e4-4934-828f-e03d89c20393",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\n",
"mistral_api_key = os.getenv(\"MISTRAL_API_KEY\") # Ensure this is set"
]
"source": ["import os\n\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\nmistral_api_key = os.getenv(\"MISTRAL_API_KEY\") # Ensure this is set"]
},
{
"cell_type": "markdown",
@@ -76,12 +69,7 @@
"id": "37b172d2-3a9d-49a8-898c-22ed0cb45c88",
"metadata": {},
"outputs": [],
"source": [
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
"os.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\"\n",
"os.environ[\"LANGCHAIN_PROJECT\"] = \"Mistral-code-gen-testing\""
]
"source": ["os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\"\nos.environ[\"LANGCHAIN_PROJECT\"] = \"Mistral-code-gen-testing\""]
},
{
"cell_type": "markdown",
@@ -99,42 +87,7 @@
"id": "a188c8ca-c053-4e6d-b7af-38a3b6b371c7",
"metadata": {},
"outputs": [],
"source": [
"# Select LLM\n",
"from langchain_core.prompts import ChatPromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain_mistralai import ChatMistralAI\n",
"\n",
"mistral_model = \"mistral-large-latest\"\n",
"llm = ChatMistralAI(model=mistral_model, temperature=0)\n",
"\n",
"# Prompt\n",
"code_gen_prompt_claude = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\n",
" \"system\",\n",
" \"\"\"You are a coding assistant. Ensure any code you provide can be executed with all required imports and variables \\n\n",
" defined. Structure your answer: 1) a prefix describing the code solution, 2) the imports, 3) the functioning code block.\n",
" \\n Here is the user question:\"\"\",\n",
" ),\n",
" (\"placeholder\", \"{messages}\"),\n",
" ]\n",
")\n",
"\n",
"\n",
"# Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
" description = \"Schema for code solutions to questions about LCEL.\"\n",
"\n",
"\n",
"# LLM\n",
"code_gen_chain = llm.with_structured_output(code, include_raw=False)"
]
"source": ["# Select LLM\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_mistralai import ChatMistralAI\n\nmistral_model = \"mistral-large-latest\"\nllm = ChatMistralAI(model=mistral_model, temperature=0)\n\n# Prompt\ncode_gen_prompt_claude = ChatPromptTemplate.from_messages(\n [\n (\n \"system\",\n \"\"\"You are a coding assistant. Ensure any code you provide can be executed with all required imports and variables \\n\n defined. Structure your answer: 1) a prefix describing the code solution, 2) the imports, 3) the functioning code block.\n \\n Here is the user question:\"\"\",\n ),\n (\"placeholder\", \"{messages}\"),\n ]\n)\n\n\n# Data model\nclass code(BaseModel):\n \"\"\"Code output\"\"\"\n\n prefix: str = Field(description=\"Description of the problem and approach\")\n imports: str = Field(description=\"Code block import statements\")\n code: str = Field(description=\"Code block not including import statements\")\n description = \"Schema for code solutions to questions about LCEL.\"\n\n\n# LLM\ncode_gen_chain = llm.with_structured_output(code, include_raw=False)"]
},
{
"cell_type": "code",
@@ -142,10 +95,7 @@
"id": "9fc0290d-5a04-4514-8664-91f9dbf2da7b",
"metadata": {},
"outputs": [],
"source": [
"question = \"Write a function for fibonacci.\"\n",
"messages = [(\"user\", question)]"
]
"source": ["question = \"Write a function for fibonacci.\"\nmessages = [(\"user\", question)]"]
},
{
"cell_type": "code",
@@ -164,11 +114,7 @@
"output_type": "execute_result"
}
],
"source": [
"# Test\n",
"result = code_gen_chain.invoke(messages)\n",
"result"
]
"source": ["# Test\nresult = code_gen_chain.invoke(messages)\nresult"]
},
{
"cell_type": "markdown",
@@ -184,28 +130,7 @@
"id": "183d77b8-f180-4815-b39f-8ef507ec0534",
"metadata": {},
"outputs": [],
"source": [
"from typing import Annotated, TypedDict\n",
"\n",
"from langgraph.graph.message import AnyMessage, add_messages\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
"\n",
" Attributes:\n",
" error : Binary flag for control flow to indicate whether test error was tripped\n",
" messages : With user question, error messages, reasoning\n",
" generation : Code solution\n",
" iterations : Number of tries\n",
" \"\"\"\n",
"\n",
" error: str\n",
" messages: Annotated[list[AnyMessage], add_messages]\n",
" generation: str\n",
" iterations: int"
]
"source": ["from typing import Annotated, TypedDict\n\nfrom langgraph.graph.message import AnyMessage, add_messages\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n error : Binary flag for control flow to indicate whether test error was tripped\n messages : With user question, error messages, reasoning\n generation : Code solution\n iterations : Number of tries\n \"\"\"\n\n error: str\n messages: Annotated[list[AnyMessage], add_messages]\n generation: str\n iterations: int"]
},
{
"cell_type": "markdown",
@@ -221,163 +146,7 @@
"id": "14bc89d1-3ca6-4847-a048-1803e0e4600e",
"metadata": {},
"outputs": [],
"source": [
"import uuid\n",
"\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"\n",
"### Parameters\n",
"max_iterations = 3\n",
"\n",
"\n",
"### Nodes\n",
"def generate(state: GraphState):\n",
" \"\"\"\n",
" Generate a code solution\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, generation\n",
" \"\"\"\n",
"\n",
" print(\"---GENERATING CODE SOLUTION---\")\n",
"\n",
" # State\n",
" messages = state[\"messages\"]\n",
" iterations = state[\"iterations\"]\n",
"\n",
" # Solution\n",
" code_solution = code_gen_chain.invoke(messages)\n",
" messages += [\n",
" (\n",
" \"assistant\",\n",
" f\"Here is my attempt to solve the problem: {code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n",
" )\n",
" ]\n",
"\n",
" # Increment\n",
" iterations = iterations + 1\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
"\n",
"\n",
"def code_check(state: GraphState):\n",
" \"\"\"\n",
" Check code\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, error\n",
" \"\"\"\n",
"\n",
" print(\"---CHECKING CODE---\")\n",
"\n",
" # State\n",
" messages = state[\"messages\"]\n",
" code_solution = state[\"generation\"]\n",
" iterations = state[\"iterations\"]\n",
"\n",
" # Get solution components\n",
" imports = code_solution.imports\n",
" code = code_solution.code\n",
"\n",
" # Check imports\n",
" try:\n",
" exec(imports)\n",
" except Exception as e:\n",
" print(\"---CODE IMPORT CHECK: FAILED---\")\n",
" error_message = [\n",
" (\n",
" \"user\",\n",
" f\"Your solution failed the import test. Here is the error: {e}. Reflect on this error and your prior attempt to solve the problem. (1) State what you think went wrong with the prior solution and (2) try to solve this problem again. Return the FULL SOLUTION. Use the code tool to structure the output with a prefix, imports, and code block:\",\n",
" )\n",
" ]\n",
" messages += error_message\n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"yes\",\n",
" }\n",
"\n",
" # Check execution\n",
" try:\n",
" combined_code = f\"{imports}\\n{code}\"\n",
" print(f\"CODE TO TEST: {combined_code}\")\n",
" # Use a shared scope for exec\n",
" global_scope = {}\n",
" exec(combined_code, global_scope)\n",
" except Exception as e:\n",
" print(\"---CODE BLOCK CHECK: FAILED---\")\n",
" error_message = [\n",
" (\n",
" \"user\",\n",
" f\"Your solution failed the code execution test: {e}) Reflect on this error and your prior attempt to solve the problem. (1) State what you think went wrong with the prior solution and (2) try to solve this problem again. Return the FULL SOLUTION. Use the code tool to structure the output with a prefix, imports, and code block:\",\n",
" )\n",
" ]\n",
" messages += error_message\n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"yes\",\n",
" }\n",
"\n",
" # No errors\n",
" print(\"---NO CODE TEST FAILURES---\")\n",
" return {\n",
" \"generation\": code_solution,\n",
" \"messages\": messages,\n",
" \"iterations\": iterations,\n",
" \"error\": \"no\",\n",
" }\n",
"\n",
"\n",
"### Conditional edges\n",
"\n",
"\n",
"def decide_to_finish(state: GraphState):\n",
" \"\"\"\n",
" Determines whether to finish.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Next node to call\n",
" \"\"\"\n",
" error = state[\"error\"]\n",
" iterations = state[\"iterations\"]\n",
"\n",
" if error == \"no\" or iterations == max_iterations:\n",
" print(\"---DECISION: FINISH---\")\n",
" return \"end\"\n",
" else:\n",
" print(\"---DECISION: RE-TRY SOLUTION---\")\n",
" return \"generate\"\n",
"\n",
"\n",
"### Utilities\n",
"\n",
"\n",
"def _print_event(event: dict, _printed: set, max_length=1500):\n",
" current_state = event.get(\"dialog_state\")\n",
" if current_state:\n",
" print(\"Currently in: \", current_state[-1])\n",
" message = event.get(\"messages\")\n",
" if message:\n",
" if isinstance(message, list):\n",
" message = message[-1]\n",
" if message.id not in _printed:\n",
" msg_repr = message.pretty_repr(html=True)\n",
" if len(msg_repr) > max_length:\n",
" msg_repr = msg_repr[:max_length] + \" ... (truncated)\"\n",
" print(msg_repr)\n",
" _printed.add(message.id)"
]
"source": ["import uuid\n\nfrom langchain_core.pydantic_v1 import BaseModel, Field\n\n### Parameters\nmax_iterations = 3\n\n\n### Nodes\ndef generate(state: GraphState):\n \"\"\"\n Generate a code solution\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation\n \"\"\"\n\n print(\"---GENERATING CODE SOLUTION---\")\n\n # State\n messages = state[\"messages\"]\n iterations = state[\"iterations\"]\n\n # Solution\n code_solution = code_gen_chain.invoke(messages)\n messages += [\n (\n \"assistant\",\n f\"Here is my attempt to solve the problem: {code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n )\n ]\n\n # Increment\n iterations = iterations + 1\n return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n\n\ndef code_check(state: GraphState):\n \"\"\"\n Check code\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, error\n \"\"\"\n\n print(\"---CHECKING CODE---\")\n\n # State\n messages = state[\"messages\"]\n code_solution = state[\"generation\"]\n iterations = state[\"iterations\"]\n\n # Get solution components\n imports = code_solution.imports\n code = code_solution.code\n\n # Check imports\n try:\n exec(imports)\n except Exception as e:\n print(\"---CODE IMPORT CHECK: FAILED---\")\n error_message = [\n (\n \"user\",\n f\"Your solution failed the import test. Here is the error: {e}. Reflect on this error and your prior attempt to solve the problem. (1) State what you think went wrong with the prior solution and (2) try to solve this problem again. Return the FULL SOLUTION. Use the code tool to structure the output with a prefix, imports, and code block:\",\n )\n ]\n messages += error_message\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"yes\",\n }\n\n # Check execution\n try:\n combined_code = f\"{imports}\\n{code}\"\n print(f\"CODE TO TEST: {combined_code}\")\n # Use a shared scope for exec\n global_scope = {}\n exec(combined_code, global_scope)\n except Exception as e:\n print(\"---CODE BLOCK CHECK: FAILED---\")\n error_message = [\n (\n \"user\",\n f\"Your solution failed the code execution test: {e}) Reflect on this error and your prior attempt to solve the problem. (1) State what you think went wrong with the prior solution and (2) try to solve this problem again. Return the FULL SOLUTION. Use the code tool to structure the output with a prefix, imports, and code block:\",\n )\n ]\n messages += error_message\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"yes\",\n }\n\n # No errors\n print(\"---NO CODE TEST FAILURES---\")\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"no\",\n }\n\n\n### Conditional edges\n\n\ndef decide_to_finish(state: GraphState):\n \"\"\"\n Determines whether to finish.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Next node to call\n \"\"\"\n error = state[\"error\"]\n iterations = state[\"iterations\"]\n\n if error == \"no\" or iterations == max_iterations:\n print(\"---DECISION: FINISH---\")\n return \"end\"\n else:\n print(\"---DECISION: RE-TRY SOLUTION---\")\n return \"generate\"\n\n\n### Utilities\n\n\ndef _print_event(event: dict, _printed: set, max_length=1500):\n current_state = event.get(\"dialog_state\")\n if current_state:\n print(\"Currently in: \", current_state[-1])\n message = event.get(\"messages\")\n if message:\n if isinstance(message, list):\n message = message[-1]\n if message.id not in _printed:\n msg_repr = message.pretty_repr(html=True)\n if len(msg_repr) > max_length:\n msg_repr = msg_repr[:max_length] + \" ... (truncated)\"\n print(msg_repr)\n _printed.add(message.id)"]
},
{
"cell_type": "code",
@@ -385,31 +154,7 @@
"id": "2dff2209-44c7-4e2c-b607-ba6675f9e45f",
"metadata": {},
"outputs": [],
"source": [
"from langgraph.checkpoint.sqlite import SqliteSaver\n",
"from langgraph.graph import END, StateGraph\n",
"\n",
"builder = StateGraph(GraphState)\n",
"\n",
"# Define the nodes\n",
"builder.add_node(\"generate\", generate) # generation solution\n",
"builder.add_node(\"check_code\", code_check) # check code\n",
"\n",
"# Build graph\n",
"builder.set_entry_point(\"generate\")\n",
"builder.add_edge(\"generate\", \"check_code\")\n",
"builder.add_conditional_edges(\n",
" \"check_code\",\n",
" decide_to_finish,\n",
" {\n",
" \"end\": END,\n",
" \"generate\": \"generate\",\n",
" },\n",
")\n",
"\n",
"memory = SqliteSaver.from_conn_string(\":memory:\")\n",
"graph = builder.compile(checkpointer=memory)"
]
"source": ["from langgraph.checkpoint.sqlite import SqliteSaver\nfrom langgraph.graph import END, StateGraph, START\n\nbuilder = StateGraph(GraphState)\n\n# Define the nodes\nbuilder.add_node(\"generate\", generate) # generation solution\nbuilder.add_node(\"check_code\", code_check) # check code\n\n# Build graph\nbuilder.add_edge(START, \"generate\")\nbuilder.add_edge(\"generate\", \"check_code\")\nbuilder.add_conditional_edges(\n \"check_code\",\n decide_to_finish,\n {\n \"end\": END,\n \"generate\": \"generate\",\n },\n)\n\nmemory = SqliteSaver.from_conn_string(\":memory:\")\ngraph = builder.compile(checkpointer=memory)"]
},
{
"cell_type": "code",
@@ -428,15 +173,7 @@
"output_type": "display_data"
}
],
"source": [
"from IPython.display import Image, display\n",
"\n",
"try:\n",
" display(Image(graph.get_graph(xray=True).draw_mermaid_png()))\n",
"except Exception:\n",
" # This requires some extra dependencies and is optional\n",
" pass"
]
"source": ["from IPython.display import Image, display\n\ntry:\n display(Image(graph.get_graph(xray=True).draw_mermaid_png()))\nexcept Exception:\n # This requires some extra dependencies and is optional\n pass"]
},
{
"cell_type": "code",
@@ -444,23 +181,7 @@
"id": "242aa2f0-2c31-462f-a958-ff9ae0cf7c62",
"metadata": {},
"outputs": [],
"source": [
"_printed = set()\n",
"thread_id = str(uuid.uuid4())\n",
"config = {\n",
" \"configurable\": {\n",
" # Checkpoints are accessed by thread_id\n",
" \"thread_id\": thread_id,\n",
" }\n",
"}\n",
"\n",
"question = \"Write a Python program that prints 'Hello, World!' to the console.\"\n",
"events = graph.stream(\n",
" {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n",
")\n",
"for event in events:\n",
" _print_event(event, _printed)"
]
"source": ["_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n \"configurable\": {\n # Checkpoints are accessed by thread_id\n \"thread_id\": thread_id,\n }\n}\n\nquestion = \"Write a Python program that prints 'Hello, World!' to the console.\"\nevents = graph.stream(\n {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n _print_event(event, _printed)"]
},
{
"cell_type": "markdown",
@@ -478,31 +199,7 @@
"id": "390b2768-f395-4aea-8b0e-9d36212a31ac",
"metadata": {},
"outputs": [],
"source": [
"_printed = set()\n",
"thread_id = str(uuid.uuid4())\n",
"config = {\n",
" \"configurable\": {\n",
" # Checkpoints are accessed by thread_id\n",
" \"thread_id\": thread_id,\n",
" }\n",
"}\n",
"\n",
"question = \"\"\"Create a Python program that checks if a given string is a palindrome. A palindrome is a word, phrase, number, or other sequence of characters that reads the same forward and backward (ignoring spaces, punctuation, and capitalization).\n",
"\n",
"Requirements:\n",
"The program should define a function is_palindrome(s) that takes a string s as input.\n",
"The function should return True if the string is a palindrome and False otherwise.\n",
"Ignore spaces, punctuation, and case differences when checking for palindromes.\n",
"\n",
"Give an example of it working on an example input word.\"\"\"\n",
"\n",
"events = graph.stream(\n",
" {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n",
")\n",
"for event in events:\n",
" _print_event(event, _printed)"
]
"source": ["_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n \"configurable\": {\n # Checkpoints are accessed by thread_id\n \"thread_id\": thread_id,\n }\n}\n\nquestion = \"\"\"Create a Python program that checks if a given string is a palindrome. A palindrome is a word, phrase, number, or other sequence of characters that reads the same forward and backward (ignoring spaces, punctuation, and capitalization).\n\nRequirements:\nThe program should define a function is_palindrome(s) that takes a string s as input.\nThe function should return True if the string is a palindrome and False otherwise.\nIgnore spaces, punctuation, and case differences when checking for palindromes.\n\nGive an example of it working on an example input word.\"\"\"\n\nevents = graph.stream(\n {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n _print_event(event, _printed)"]
},
{
"cell_type": "markdown",
@@ -520,26 +217,7 @@
"id": "0a3f946b-e2f2-44d9-905b-09f36980cf9f",
"metadata": {},
"outputs": [],
"source": [
"_printed = set()\n",
"thread_id = str(uuid.uuid4())\n",
"config = {\n",
" \"configurable\": {\n",
" # Checkpoints are accessed by thread_id\n",
" \"thread_id\": thread_id,\n",
" }\n",
"}\n",
"\n",
"question = \"\"\"Write a program that prints the numbers from 1 to 100. \n",
"But for multiples of three, print \"Fizz\" instead of the number, and for the multiples of five, print \"Buzz\". \n",
"For numbers which are multiples of both three and five, print \"FizzBuzz\".\"\"\"\n",
"\n",
"events = graph.stream(\n",
" {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n",
")\n",
"for event in events:\n",
" _print_event(event, _printed)"
]
"source": ["_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n \"configurable\": {\n # Checkpoints are accessed by thread_id\n \"thread_id\": thread_id,\n }\n}\n\nquestion = \"\"\"Write a program that prints the numbers from 1 to 100. \nBut for multiples of three, print \"Fizz\" instead of the number, and for the multiples of five, print \"Buzz\". \nFor numbers which are multiples of both three and five, print \"FizzBuzz\".\"\"\"\n\nevents = graph.stream(\n {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n _print_event(event, _printed)"]
},
{
"cell_type": "markdown",
@@ -557,37 +235,7 @@
"id": "2bb883df-540b-46ab-9415-fe27db68456f",
"metadata": {},
"outputs": [],
"source": [
"import uuid\n",
"\n",
"_printed = set()\n",
"thread_id = str(uuid.uuid4())\n",
"config = {\n",
" \"configurable\": {\n",
" # Checkpoints are accessed by thread_id\n",
" \"thread_id\": thread_id,\n",
" }\n",
"}\n",
"\n",
"question = \"\"\"I want to vectorize a function\n",
"\n",
" frame = np.zeros((out_h, out_w, 3), dtype=np.uint8)\n",
" for i, val1 in enumerate(rows):\n",
" for j, val2 in enumerate(cols):\n",
" for j, val3 in enumerate(ch):\n",
" # Assuming you want to store the pair as tuples in the matrix\n",
" frame[i, j, k] = image[val1, val2, val3]\n",
"\n",
" out.write(np.array(frame))\n",
"\n",
"with a simple numpy function that does something like this what is it called. Show me a test case with this working.\"\"\"\n",
"\n",
"events = graph.stream(\n",
" {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n",
")\n",
"for event in events:\n",
" _print_event(event, _printed)"
]
"source": ["import uuid\n\n_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n \"configurable\": {\n # Checkpoints are accessed by thread_id\n \"thread_id\": thread_id,\n }\n}\n\nquestion = \"\"\"I want to vectorize a function\n\n frame = np.zeros((out_h, out_w, 3), dtype=np.uint8)\n for i, val1 in enumerate(rows):\n for j, val2 in enumerate(cols):\n for j, val3 in enumerate(ch):\n # Assuming you want to store the pair as tuples in the matrix\n frame[i, j, k] = image[val1, val2, val3]\n\n out.write(np.array(frame))\n\nwith a simple numpy function that does something like this what is it called. Show me a test case with this working.\"\"\"\n\nevents = graph.stream(\n {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n _print_event(event, _printed)"]
},
{
"cell_type": "markdown",
@@ -605,34 +253,7 @@
"id": "ee05da1f-c272-405d-8a7b-552cfc3106e1",
"metadata": {},
"outputs": [],
"source": [
"_printed = set()\n",
"thread_id = str(uuid.uuid4())\n",
"config = {\n",
" \"configurable\": {\n",
" # Checkpoints are accessed by thread_id\n",
" \"thread_id\": thread_id,\n",
" }\n",
"}\n",
"\n",
"question = \"\"\"Create a Python program that allows two players to play a game of Tic-Tac-Toe. The game should be played on a 3x3 grid. The program should:\n",
"\n",
"- Allow players to take turns to input their moves.\n",
"- Check for invalid moves (e.g., placing a marker on an already occupied space).\n",
"- Determine and announce the winner or if the game ends in a draw.\n",
"\n",
"Requirements:\n",
"- Use a 2D list to represent the Tic-Tac-Toe board.\n",
"- Use functions to modularize the code.\n",
"- Validate player input.\n",
"- Check for win conditions and draw conditions after each move.\"\"\"\n",
"\n",
"events = graph.stream(\n",
" {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n",
")\n",
"for event in events:\n",
" _print_event(event, _printed)"
]
"source": ["_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n \"configurable\": {\n # Checkpoints are accessed by thread_id\n \"thread_id\": thread_id,\n }\n}\n\nquestion = \"\"\"Create a Python program that allows two players to play a game of Tic-Tac-Toe. The game should be played on a 3x3 grid. The program should:\n\n- Allow players to take turns to input their moves.\n- Check for invalid moves (e.g., placing a marker on an already occupied space).\n- Determine and announce the winner or if the game ends in a draw.\n\nRequirements:\n- Use a 2D list to represent the Tic-Tac-Toe board.\n- Use functions to modularize the code.\n- Validate player input.\n- Check for win conditions and draw conditions after each move.\"\"\"\n\nevents = graph.stream(\n {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n _print_event(event, _printed)"]
},
{
"cell_type": "markdown",
@@ -650,7 +271,7 @@
"id": "814fc2a4-8e5b-4faa-8f52-3977226bd09a",
"metadata": {},
"outputs": [],
"source": []
"source": [""]
}
],
"metadata": {