diff --git a/examples/code_assistant/langgraph_code_assistant.ipynb b/examples/code_assistant/langgraph_code_assistant.ipynb index 608804ab0..264f36d70 100644 --- a/examples/code_assistant/langgraph_code_assistant.ipynb +++ b/examples/code_assistant/langgraph_code_assistant.ipynb @@ -47,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "id": "c2eb35d1-4990-47dc-a5c4-208bae588a82", "metadata": {}, "outputs": [], @@ -92,7 +92,6 @@ "outputs": [], "source": [ "from langchain_openai import ChatOpenAI\n", - "from langchain_anthropic import ChatAnthropic\n", "from langchain_core.prompts import ChatPromptTemplate\n", "from langchain_core.pydantic_v1 import BaseModel, Field\n", "\n", @@ -126,14 +125,18 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "id": "cd30b67d-96db-4e51-a540-ae23fcc1f878", "metadata": {}, "outputs": [], "source": [ + "from langchain_anthropic import ChatAnthropic\n", + "from langchain_core.prompts import ChatPromptTemplate\n", + "from langchain_core.pydantic_v1 import BaseModel, Field\n", + "\n", "### Anthropic\n", "\n", - "# Important for getting tool use\n", + "# Prompt to enforce tool use\n", "code_gen_prompt_claude = ChatPromptTemplate.from_messages(\n", " [(\"system\",\"\"\" You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n", " Here is the LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user question based on the \\n \n", @@ -142,8 +145,19 @@ " Invoke the code tool to structure the output correctly. \\n Here is the user question:\"\"\",),\n", " (\"placeholder\", \"{messages}\"),])\n", "\n", + "# Data model\n", + "class code(BaseModel):\n", + " \"\"\"Code output\"\"\"\n", + "\n", + " prefix: str = Field(description=\"Description of the problem and approach\")\n", + " imports: str = Field(description=\"Code block import statements\")\n", + " code: str = Field(description=\"Code block not including import statements\")\n", + " description = \"Schema for code solutions to questions about LCEL.\"\n", + "\n", + "\n", "# LLM\n", - "expt_llm = \"claude-3-haiku-20240307\" # claude-3-opus-20240229\n", + "# expt_llm = \"claude-3-haiku-20240307\" \n", + "expt_llm = \"claude-3-opus-20240229\" \n", "llm = ChatAnthropic(\n", " model=expt_llm,\n", " default_headers={\"anthropic-beta\": \"tools-2024-04-04\"},\n", @@ -177,14 +191,14 @@ "code_chain_claude_raw = code_gen_prompt_claude | structured_llm_claude | check_claude_output\n", "\n", "def insert_errors(inputs):\n", - " \"\"\"Insert errors in the messages\"\"\"\n", + " \"\"\"Insert errors for tool parsing in the messages\"\"\"\n", " \n", " # Get errors\n", " error = inputs[\"error\"]\n", " messages = inputs[\"messages\"]\n", " messages += [\n", " (\n", - " \"user\",\n", + " \"assistant\",\n", " f\"Retry. You are required to fix the parsing errors: {error} \\n\\n You must invoke the provided tool.\",\n", " )\n", " ]\n", @@ -196,11 +210,34 @@ "# This will be run as a fallback chain\n", "fallback_chain = insert_errors | code_chain_claude_raw\n", "N = 3 # Max re-tries\n", - "code_gen_chain = code_chain_claude_raw.with_fallbacks(fallbacks=[fallback_chain] * N, exception_key=\"error\")\n", + "code_gen_chain_re_try = code_chain_claude_raw.with_fallbacks(fallbacks=[fallback_chain] * N, exception_key=\"error\")\n", "\n", + "def parse_output(solution):\n", + " \"\"\"When we add 'include_raw=True' to structured output, \n", + " it will return a dict w 'raw', 'parsed', 'parsing_error'. \"\"\"\n", + " \n", + " return solution['parsed']\n", + "\n", + "# Wtih re-try\n", + "code_gen_chain = code_gen_chain_re_try | parse_output\n", + "\n", + "# No re-try\n", + "code_gen_chain = code_gen_prompt_claude | structured_llm_claude | parse_output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f14750f-dddc-485b-ba29-5392cdf4ba43", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ "# Test\n", "question = \"How do I build a RAG chain in LCEL?\"\n", - "# solution = code_gen_chain_claude.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})" + "solution = code_gen_chain.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})\n", + "solution" ] }, { @@ -215,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 4, "id": "c185f1a2-e943-4bed-b833-4243c9c64092", "metadata": {}, "outputs": [], @@ -251,7 +288,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "b70e8301-63ae-4f7e-ad8f-c9a052fe3566", "metadata": {}, "outputs": [], @@ -291,7 +328,7 @@ "\n", " # We have been routed back to generation with an error\n", " if error == \"yes\":\n", - " messages += [(\"user\",\"Now, try again. Be sure to structure your answer with a prefix, imports, and code block:\")]\n", + " messages += [(\"user\",\"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\")]\n", " \n", " # Solution\n", " code_solution = code_gen_chain.invoke({\"context\": concatenated_content, \"messages\" : messages})\n", @@ -329,7 +366,7 @@ " exec(imports)\n", " except Exception as e:\n", " print(\"---CODE IMPORT CHECK: FAILED---\")\n", - " error_message = [(\"assistant\", f\"Your solution failed the import test: {e}\")]\n", + " error_message = [(\"user\", f\"Your solution failed the import test: {e}\")]\n", " messages += error_message\n", " return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"yes\"}\n", " \n", @@ -338,7 +375,7 @@ " exec(imports + \"\\n\" + code)\n", " except Exception as e:\n", " print(\"---CODE BLOCK CHECK: FAILED---\")\n", - " error_message = [(\"assistant\", f\"Your solution failed the code execution test: {e}\")]\n", + " error_message = [(\"user\", f\"Your solution failed the code execution test: {e}\")]\n", " messages += error_message\n", " return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"yes\"}\n", " \n", @@ -402,7 +439,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "f66b4e00-4731-42c8-bc38-72dd0ff7c92c", "metadata": {}, "outputs": [], @@ -465,7 +502,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "678e8954-56b5-4cc6-be26-f7f2a060b242", "metadata": {}, "outputs": [], @@ -496,7 +533,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "455a34ea-52cb-4ae5-9f4a-7e4a08cd0c09", "metadata": {}, "outputs": [], @@ -531,7 +568,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "c8fa6bcb-b245-4422-b79a-582cd8a7d7ea", "metadata": {}, "outputs": [], @@ -551,7 +588,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "d9c57468-97f6-47d6-a5e9-c09b53bfdd83", "metadata": {}, "outputs": [], @@ -598,7 +635,7 @@ " data=dataset_name,\n", " evaluators=code_evalulator,\n", " experiment_prefix=f\"test-with-langgraph-{expt_llm}-{flag}\",\n", - " max_concurrency=10,\n", + " max_concurrency=2,\n", " metadata={\n", " \"llm\": expt_llm,\n", " \"feedback\": flag,\n", @@ -611,11 +648,11 @@ "id": "d69da747-b4ea-455d-9314-60c3d9d30549", "metadata": {}, "source": [ - "Results:\n", + "`Results:`\n", "\n", - "LangGraph w/o reflection performs the best by a wide margin.\n", - "\n", - "Reflection may confuse the re-try, and needs prompt engineering.\n", + "* `LangGraph outperforms base case`: adding re-try loop improve performance\n", + "* `Reflection did not help`: reflection prior to re-try regression vs just passing errors directly back to the LLM\n", + "* `GPT-4 outperforms Claude3`: Claude3 had 3 and 1 run fail due to tool-use error for Opus and Haiku, repspectively\n", "\n", "https://smith.langchain.com/public/78a3d858-c811-4e46-91cb-0f10ef56260b/d" ] @@ -623,7 +660,7 @@ { "cell_type": "code", "execution_count": null, - "id": "96f85440-3622-4ae3-ae6f-7c0613466ffb", + "id": "a42333c3-c098-4576-ae2a-0258de64ece2", "metadata": {}, "outputs": [], "source": []