diff --git a/examples/code_assistant/langgraph_code_assistant.ipynb b/examples/code_assistant/langgraph_code_assistant.ipynb
index 6cf6e2501..988c3029b 100644
--- a/examples/code_assistant/langgraph_code_assistant.ipynb
+++ b/examples/code_assistant/langgraph_code_assistant.ipynb
@@ -27,7 +27,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"id": "e3900420",
"metadata": {},
"outputs": [],
@@ -47,7 +47,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"id": "c2eb35d1-4990-47dc-a5c4-208bae588a82",
"metadata": {},
"outputs": [],
@@ -79,12 +79,14 @@
"\n",
"### Code solution\n",
"\n",
+ "We can consider [Claude3](https://docs.anthropic.com/claude/docs/models-overview) variants, too.\n",
+ "\n",
"We de-couple code solution and code formatting so that any LLM can be used for code solution. "
]
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": null,
"id": "3ba3df70-f6b4-4ea5-a210-e10944960bc6",
"metadata": {},
"outputs": [],
@@ -104,41 +106,6 @@
" (\"placeholder\", \"{messages}\")]\n",
")\n",
"\n",
- "# code_gen_llm = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\")\n",
- "code_gen_llm = ChatAnthropic(temperature=0, model='claude-3-opus-20240229')\n",
- "\n",
- "code_gen_chain = code_gen_prompt | code_gen_llm | StrOutputParser()\n",
- "question = \"How do I build a RAG chain in LCEL?\"\n",
- "solution = code_gen_chain.invoke({\"context\" : concatenated_content, \"messages\" : [(\"user\",question)]})"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "eb01dcde-b446-4fae-90af-a303222f90e7",
- "metadata": {},
- "source": [
- "### Formatted code\n",
- "\n",
- "The structured output generation is handled in a seperate step."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 17,
- "id": "4ad38291-807b-43e2-952b-8d9007617630",
- "metadata": {},
- "outputs": [],
- "source": [
- "from langchain_openai import ChatOpenAI\n",
- "from langchain_core.pydantic_v1 import BaseModel, Field\n",
- "\n",
- "# Prompt\n",
- "prompt = ChatPromptTemplate.from_messages(\n",
- " [(\"system\",\"\"\"You are an expert a code formatting, strating with a code solution \\n\n",
- " Structure the solution in three parts with a prefix that defines the problem, then \\n \n",
- " list the imports, and finally list the functioning code block.\"\"\" ),\n",
- " (\"user\", \"Here is the code solution: {code}\"),]) \n",
- " \n",
"# Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
@@ -147,29 +114,19 @@
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
"\n",
- "# Formatter \n",
- "llm = ChatOpenAI(model=\"gpt-4-0125-preview\", temperature=0)\n",
- "llm_formatter = llm.with_structured_output(code)\n",
- "structured_code_formatter = prompt | llm_formatter\n",
- "output = structured_code_formatter.invoke([(\"code\",solution)])"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "00d6a0b4-9b7e-4067-8622-832201ad9b56",
- "metadata": {},
- "source": [
- "## Parameters"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 18,
- "id": "01598aee-f272-4d72-aefc-2cb15dcbfdfc",
- "metadata": {},
- "outputs": [],
- "source": [
- "max_iterations = 3"
+ "'''\n",
+ "expt_llm = \"claude3-opus\"\n",
+ "code_gen_llm = ChatAnthropic(temperature=0, model='claude-3-opus-20240229')\n",
+ "expt_llm = \"claude3-haiku\"\n",
+ "code_gen_llm = ChatAnthropic(temperature=0, model=\"claude-3-haiku-20240307\")\n",
+ "'''\n",
+ "\n",
+ "expt_llm = \"gpt-4-0125-preview\"\n",
+ "llm = ChatOpenAI(temperature=0, model=expt_llm)\n",
+ "code_gen_chain = code_gen_prompt | llm.with_structured_output(code)\n",
+ "question = \"How do I build a RAG chain in LCEL?\"\n",
+ "solution = code_gen_chain.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})\n",
+ "solution"
]
},
{
@@ -184,7 +141,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"id": "c185f1a2-e943-4bed-b833-4243c9c64092",
"metadata": {},
"outputs": [],
@@ -220,7 +177,7 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": null,
"id": "b70e8301-63ae-4f7e-ad8f-c9a052fe3566",
"metadata": {},
"outputs": [],
@@ -230,6 +187,14 @@
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain_core.runnables import RunnablePassthrough\n",
"\n",
+ "### Parameter\n",
+ "\n",
+ "# Max tries\n",
+ "max_iterations = 3\n",
+ "# Reflect\n",
+ "# flag = 'reflect'\n",
+ "flag = 'do not reflect'\n",
+ " \n",
"### Nodes\n",
"\n",
"def generate(state: GraphState):\n",
@@ -248,18 +213,20 @@
" # State\n",
" messages = state[\"messages\"]\n",
" iterations = state[\"iterations\"]\n",
- " \n",
- " # Solution\n",
- " solution = code_gen_chain.invoke({\"context\" : concatenated_content, \"messages\" : messages})\n",
+ " error = state[\"error\"]\n",
"\n",
- " # Structured output\n",
- " code_solution = structured_code_formatter.invoke([(\"code\",solution)])\n",
+ " # We have been routed back to generation with an error\n",
+ " if error == \"yes\":\n",
+ " messages += [(\"user\",\"Now, try again. Be sure to structure your answer with a prefix, imports, and code block:\")]\n",
+ " \n",
+ " # Solution\n",
+ " code_solution = code_gen_chain.invoke({\"context\": concatenated_content, \"messages\" : messages})\n",
+ " messages += [(\"assistant\",f\"{code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\")]\n",
" \n",
" # Increment\n",
" iterations = iterations + 1\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
"\n",
- "\n",
"def code_check(state: GraphState):\n",
" \"\"\"\n",
" Check code\n",
@@ -273,7 +240,7 @@
"\n",
" print(\"---CHECKING CODE---\")\n",
" \n",
- " ## State\n",
+ " # State\n",
" messages = state[\"messages\"]\n",
" code_solution = state[\"generation\"]\n",
" iterations = state[\"iterations\"]\n",
@@ -288,7 +255,7 @@
" exec(imports)\n",
" except Exception as e:\n",
" print(\"---CODE IMPORT CHECK: FAILED---\")\n",
- " error_message = [(\"user\", f\"Import Test Failure: You are required to fix the import error: {e}\")]\n",
+ " error_message = [(\"assistant\", f\"Your solution failed the import test: {e}\")]\n",
" messages += error_message\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"yes\"}\n",
" \n",
@@ -297,14 +264,14 @@
" exec(imports + \"\\n\" + code)\n",
" except Exception as e:\n",
" print(\"---CODE BLOCK CHECK: FAILED---\")\n",
- " error_message = [(\"user\", f\"Execution Test Failure: You are required to fix the code execution error: {e}\")]\n",
+ " error_message = [(\"assistant\", f\"Your solution failed the code execution test: {e}\")]\n",
+ " messages += error_message\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"yes\"}\n",
" \n",
" # No errors\n",
" print(\"---NO CODE TEST FAILURES---\")\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations, \"error\": \"no\"}\n",
"\n",
- "\n",
"def reflect(state: GraphState):\n",
" \"\"\"\n",
" Reflect on errors\n",
@@ -321,19 +288,18 @@
" # State\n",
" messages = state[\"messages\"]\n",
" iterations = state[\"iterations\"]\n",
+ " code_solution = state[\"generation\"]\n",
"\n",
" # Prompt reflection\n",
" reflection_message = [(\"user\", \"\"\"You tried to solve this problem and failed a unit test. Reflect on this failure\n",
- " given the provided documentation. Carefully write suggestions based on the \n",
+ " given the provided documentation. Write a few key suggestions based on the \n",
" documentation to avoid making this mistake again.\"\"\")]\n",
- " messages += reflection_message\n",
" \n",
- " # Suggesting\n",
+ " # Add reflection\n",
" reflections = code_gen_chain.invoke({\"context\" : concatenated_content, \"messages\" : messages})\n",
- " messages += reflections\n",
+ " messages += [(\"assistant\" , f\"Here are reflections on the error: {reflections}\")]\n",
" return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
"\n",
- "\n",
"### Edges\n",
"\n",
"def decide_to_finish(state: GraphState):\n",
@@ -354,12 +320,15 @@
" return \"end\"\n",
" else:\n",
" print(\"---DECISION: RE-TRY SOLUTION---\")\n",
- " return \"reflect\" # Or, directly back to generate"
+ " if flag == 'reflect':\n",
+ " return \"reflect\"\n",
+ " else:\n",
+ " return \"generate\""
]
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": null,
"id": "f66b4e00-4731-42c8-bc38-72dd0ff7c92c",
"metadata": {},
"outputs": [],
@@ -382,6 +351,7 @@
" {\n",
" \"end\": END,\n",
" \"reflect\": \"reflect\",\n",
+ " \"generate\": \"generate\",\n",
" },\n",
")\n",
"workflow.add_edge(\"reflect\", \"generate\")\n",
@@ -390,36 +360,12 @@
},
{
"cell_type": "code",
- "execution_count": 30,
- "id": "c4b1b35b-c8bd-4c68-a4b1-f09a7dfc8707",
+ "execution_count": null,
+ "id": "9bcaafe4-ddcf-4fab-8620-2d9b6c508f98",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "---GENERATING CODE SOLUTION---\n",
- "---CHECKING CODE---\n",
- "---NO CODE TEST FAILURES---\n",
- "---DECISION: FINISH---\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "{'error': 'no',\n",
- " 'messages': [('user', 'How do I build a RAG chain in LCEL?')],\n",
- " 'generation': code(prefix=\"Problem: Building a Retrieval-Augmented Generation (RAG) Chain using LangChain Expression Language (LCEL).\\n\\nApproach: To construct a RAG chain in LCEL, it's essential to integrate a retriever, a prompt template, a language model, and an output parser. The process involves creating a retriever to fetch relevant documents based on a query, defining a prompt template that incorporates the retrieved context and a question, instantiating a language model, and creating a chain by connecting the retriever, prompt, model, and output parser. Finally, the chain is invoked with a question to obtain the answer.\", imports='from langchain_community.vectorstores import FAISS\\nfrom langchain_core.output_parsers import StrOutputParser\\nfrom langchain_core.prompts import ChatPromptTemplate\\nfrom langchain_core.runnables import RunnablePassthrough\\nfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings', code='# Load documents into a vector store\\nvectorstore = FAISS.from_texts(\\n [\"harrison worked at kensho\"], embedding=OpenAIEmbeddings()\\n)\\nretriever = vectorstore.as_retriever()\\n\\n# Define prompt template\\ntemplate = \"\"\"Answer the question based only on the following context:\\n{context}\\nQuestion: {question}\"\"\"\\nprompt = ChatPromptTemplate.from_template(template)\\n\\n# Instantiate model and output parser\\nmodel = ChatOpenAI()\\noutput_parser = StrOutputParser()\\n\\n# Create RAG chain \\nchain = (\\n {\"context\": retriever, \"question\": RunnablePassthrough()}\\n | prompt \\n | model\\n | output_parser\\n)\\n\\n# Run the chain\\nchain.invoke(\"where did harrison work?\")'),\n",
- " 'iterations': 1}"
- ]
- },
- "execution_count": 30,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
- "question = \"How do I build a RAG chain in LCEL?\"\n",
+ "question = \"How can I directly pass a string to a runnable and use it to construct the input needed for my prompt?\"\n",
"app.invoke({\"messages\":[(\"user\",question)],\"iterations\":0})"
]
},
@@ -445,7 +391,7 @@
},
{
"cell_type": "code",
- "execution_count": 32,
+ "execution_count": null,
"id": "678e8954-56b5-4cc6-be26-f7f2a060b242",
"metadata": {},
"outputs": [],
@@ -476,7 +422,7 @@
},
{
"cell_type": "code",
- "execution_count": 33,
+ "execution_count": null,
"id": "455a34ea-52cb-4ae5-9f4a-7e4a08cd0c09",
"metadata": {},
"outputs": [],
@@ -511,14 +457,14 @@
},
{
"cell_type": "code",
- "execution_count": 34,
+ "execution_count": null,
"id": "c8fa6bcb-b245-4422-b79a-582cd8a7d7ea",
"metadata": {},
"outputs": [],
"source": [
"def predict_base_case(example: dict):\n",
" \"\"\" Context stuffing \"\"\"\n",
- " solution = code_gen_chain.invoke({\"context\" : concatenated_content, \"messages\" : [(\"user\",example[\"question\"])] })\n",
+ " solution = code_gen_chain.invoke({\"context\" : concatenated_content, \"messages\" : [(\"user\",example[\"question\"])]})\n",
" solution_structured = structured_code_formatter.invoke([(\"code\",solution)])\n",
" return {\"imports\": solution_structured.imports, \"code\": solution_structured.code}\n",
"\n",
@@ -532,64 +478,9 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "2dacccf0-d73f-4017-aaf0-9806ffe5bd2c",
+ "id": "d9c57468-97f6-47d6-a5e9-c09b53bfdd83",
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/var/folders/l9/bpjxdmfx7lvd1fbdjn38y5dh0000gn/T/ipykernel_43078/1906326331.py:10: UserWarning: Function evaluate is in beta.\n",
- " experiment_results = evaluate(\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "View the evaluation results for experiment: 'test-without-langgraph:1f3259c' at:\n",
- "https://smith.langchain.com/o/1fa8b1f4-fcb9-4072-9aa9-983e35ad61b8/datasets/76bc1d66-8cb6-4f6d-b633-7cd077937e46/compare?selectedSessions=1c92918c-b111-412d-b845-a96f2ee37f7c\n",
- "\n",
- "\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "981c40cd7ce34b97a06d8f4daa339710",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "0it [00:00, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n",
- "Error running target function: Error code: 429 - {'type': 'error', 'error': {'type': 'rate_limit_error', 'message': 'Number of concurrent connections has exceeded your rate limit. Please try again later or contact sales at https://www.anthropic.com/contact-sales to discuss your options for a rate limit increase.'}}\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"from langsmith.evaluation import evaluate\n",
"\n",
@@ -597,358 +488,68 @@
"code_evalulator = [check_import,check_execution]\n",
"\n",
"# Dataset\n",
- "dataset_name = \"test-LCEL-code-gen\"\n",
- "\n",
+ "dataset_name = \"test-LCEL-code-gen\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2dacccf0-d73f-4017-aaf0-9806ffe5bd2c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
"# Run base case\n",
- "experiment_results = evaluate(\n",
+ "experiment_results_ = evaluate(\n",
" predict_base_case,\n",
" data=dataset_name,\n",
" evaluators=code_evalulator,\n",
- " experiment_prefix=\"test-without-langgraph\",\n",
+ " experiment_prefix=f\"test-without-langgraph-{expt_llm}\", \n",
+ " max_concurrency=2,\n",
" metadata={\n",
- " \"variant\": \"Claude3\",\n",
+ " \"llm\": expt_llm,\n",
" },\n",
- ")\n",
- "\n",
- "# Run with langgraph\n",
- "experiment_results = evaluate(\n",
- " predict_langgraph,\n",
- " data=dataset_name,\n",
- " evaluators=code_evalulator,\n",
- " experiment_prefix=\"test-with-langgraph\",\n",
- " metadata={\n",
- " \"variant\": \"Claude3\",\n",
- " },\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "5caa3a37-5fd2-4e4c-b310-577c239f9d61",
- "metadata": {},
- "source": [
- "## TODO: Clean This Later ## \n",
- "\n",
- "Compute standard error across 4 trials."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 14,
- "id": "14f8484b-9d57-4132-8801-74a4067f97db",
- "metadata": {},
- "outputs": [],
- "source": [
- "# You will have to update these to match the tests you ran.\n",
- "# The test name can be found at langgraph_results[\"project_name\"]\n",
- "langgraph = [\n",
- " \"80db-context-stuffing-with-langgraph\",\n",
- " \"060c-context-stuffing-with-langgraph\",\n",
- " \"93cd-context-stuffing-with-langgraph\",\n",
- " \"60ef-context-stuffing-with-langgraph\",\n",
- "]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "id": "d19deeb6-9fc3-46b0-affb-5baaef4e9bba",
- "metadata": {},
- "outputs": [],
- "source": [
- "no_langgraph = [\n",
- " \"b493-context-stuffing-no-langgraph\",\n",
- " \"eb8a-context-stuffing-no-langgraph\",\n",
- " \"b88c-context-stuffing-no-langgraph\",\n",
- " \"0aaa-context-stuffing-no-langgraph\",\n",
- "]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 39,
- "id": "d6340773-ca0b-4320-b841-093bbfb1161a",
- "metadata": {},
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "\n",
- "\n",
- "def prepare_dataframe(project, trial_number, chain):\n",
- " df = client.get_test_results(project_name=project)\n",
- " df = df.dropna(subset=[\"feedback.check_execution\", \"feedback.check_import\"])\n",
- " df = df[[\"input.question\", \"feedback.check_execution\", \"feedback.check_import\"]]\n",
- " df[\"trial #\"] = trial_number\n",
- " df[\"chain\"] = chain\n",
- " return df\n",
- "\n",
- "\n",
- "# Prepare each dataframe\n",
- "dfs_chain1 = [\n",
- " prepare_dataframe(project, i + 1, \"LangGraph\")\n",
- " for i, project in enumerate(langgraph)\n",
- "]\n",
- "dfs_chain2 = [\n",
- " prepare_dataframe(project, i + 1, \"No LangGraph\")\n",
- " for i, project in enumerate(no_langgraph)\n",
- "]\n",
- "\n",
- "# Combine all dataframes\n",
- "final_df = pd.concat(dfs_chain1 + dfs_chain2, ignore_index=True)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 42,
- "id": "30dd9b44-b23f-4709-a993-f1e6873ff3f2",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "chain\n",
- "LangGraph 78\n",
- "No LangGraph 79\n",
- "dtype: int64"
- ]
- },
- "execution_count": 42,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "final_df.groupby(\"chain\").size()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 40,
- "id": "2c4e226c-511b-41fa-b850-2a0a3f67a44d",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- "\n",
- "
\n",
- " \n",
- " \n",
- " | \n",
- " Fraction Imports Correct | \n",
- " Fraction Execution Correct | \n",
- " Imports Correct Std Error | \n",
- " Execution Correct Std Error | \n",
- "
\n",
- " \n",
- " | chain | \n",
- " | \n",
- " | \n",
- " | \n",
- " | \n",
- "
\n",
- " \n",
- " \n",
- " \n",
- " | LangGraph | \n",
- " 1.000000 | \n",
- " 0.807692 | \n",
- " 0.000000 | \n",
- " 0.044625 | \n",
- "
\n",
- " \n",
- " | No LangGraph | \n",
- " 0.987342 | \n",
- " 0.556962 | \n",
- " 0.012578 | \n",
- " 0.055888 | \n",
- "
\n",
- " \n",
- "
\n",
- "