WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-09-10 19:57:52 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add cloneable link in langgraph code assistant notebook

Browse Source 
Add cloneable link in langgraph code assistant notebook
This commit is contained in:
William FH
2024-03-06 17:34:03 -08:00
committed by GitHub
parent 9661514155 54e367a628
commit 4569be0e02
1 changed files with 217 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/code_assistant/langgraph_code_assistant.ipynb
+217 -139
View File
@@ -1,15 +1,5 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "10291bbc-be96-4d65-8b57-e39950332a21",
"metadata": {},
"outputs": [],
"source": [
"! pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph faiss-cpu"
]
},
{
"attachments": {
"15ffc4e1-0f2b-49fb-99c9-2bbda7643172.png": {
@@ -51,6 +41,17 @@
"![Screenshot 2024-02-16 at 11.43.52 AM.png](attachment:fb3f0be0-4884-4ad2-b9b3-cf92cfc51273.png)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "e3900420",
"metadata": {},
"outputs": [],
"source": [
"%%capture --no-stderr\n",
"%pip install -U langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph faiss-cpu"
]
},
{
"cell_type": "markdown",
"id": "38330223-d8c8-4156-82b6-93e63343bc01",
@@ -63,7 +64,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 3,
"id": "c2eb35d1-4990-47dc-a5c4-208bae588a82",
"metadata": {},
"outputs": [],
@@ -71,7 +72,7 @@
"from bs4 import BeautifulSoup as Soup\n",
"from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n",
"\n",
"# LCEL docs \n",
"# LCEL docs\n",
"url = \"https://python.langchain.com/docs/expression_language/\"\n",
"loader = RecursiveUrlLoader(\n",
" url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n",
@@ -92,7 +93,7 @@
")\n",
"docs_sq = loader.load()\n",
"\n",
"# Add \n",
"# Add\n",
"docs.extend([*docs_pydantic, *docs_sq])\n",
"\n",
"# Sort the list based on the URLs in 'metadata' -> 'source'\n",
@@ -117,15 +118,13 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 4,
"id": "c185f1a2-e943-4bed-b833-4243c9c64092",
"metadata": {},
"outputs": [],
"source": [
"from typing import Dict, TypedDict\n",
"\n",
"from langchain_core.messages import BaseMessage\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
@@ -156,17 +155,19 @@
"outputs": [],
"source": [
"from operator import itemgetter\n",
"from langchain_openai import ChatOpenAI\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain_core.runnables import RunnablePassthrough\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain.output_parsers.openai_tools import PydanticToolsParser\n",
"from langchain_core.utils.function_calling import convert_to_openai_tool\n",
"\n",
"def generate(state):\n",
"from langchain.output_parsers.openai_tools import PydanticToolsParser\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain_core.runnables import RunnablePassthrough\n",
"from langchain_core.utils.function_calling import convert_to_openai_tool\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"\n",
"def generate(state: GraphState):\n",
" \"\"\"\n",
" Generate a code solution based on LCEL docs and the input question \n",
" with optional feedback from code execution tests \n",
" Generate a code solution based on LCEL docs and the input question\n",
" with optional feedback from code execution tests\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
@@ -174,34 +175,35 @@
" Returns:\n",
" state (dict): New key added to state, documents, that contains retrieved documents\n",
" \"\"\"\n",
" \n",
"\n",
" ## State\n",
" state_dict = state[\"keys\"]\n",
" question = state_dict[\"question\"]\n",
" iter = state_dict[\"iterations\"]\n",
" \n",
"\n",
" ## Data model\n",
" class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
" \n",
"\n",
" ## LLM\n",
" model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
" \n",
"\n",
" # Tool\n",
" code_tool_oai = convert_to_openai_tool(code)\n",
" \n",
"\n",
" # LLM with tool and enforce invocation\n",
" llm_with_tool = model.bind(\n",
" tools=[code_tool_oai],\n",
" tool_choice={\"type\": \"function\", \"function\": {\"name\": \"code\"}},\n",
" )\n",
" \n",
"\n",
" # Parser\n",
" parser_tool = PydanticToolsParser(tools=[code])\n",
" \n",
"\n",
" ## Prompt\n",
" template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is a full set of LCEL documentation: \n",
@@ -217,11 +219,11 @@
" ## Generation\n",
" if \"error\" in state_dict:\n",
" print(\"---RE-GENERATE SOLUTION w/ ERROR FEEDBACK---\")\n",
" \n",
"\n",
" error = state_dict[\"error\"]\n",
" code_solution = state_dict[\"generation\"]\n",
" \n",
" # Udpate prompt \n",
"\n",
" # Udpate prompt\n",
" addendum = \"\"\" \\n --- --- --- \\n You previously tried to solve this problem. \\n Here is your solution: \n",
" \\n --- --- --- \\n {generation} \\n --- --- --- \\n Here is the resulting error from code \n",
" execution: \\n --- --- --- \\n {error} \\n --- --- --- \\n Please re-try to answer this. \n",
@@ -229,35 +231,35 @@
" And finally list the functioning code block. Structure your answer with a description of \n",
" the code solution. \\n Then list the imports. And finally list the functioning code block. \n",
" \\n Here is the user question: \\n --- --- --- \\n {question}\"\"\"\n",
" template = template + addendum\n",
" template = template + addendum\n",
"\n",
" # Prompt \n",
" # Prompt\n",
" prompt = PromptTemplate(\n",
" template=template,\n",
" input_variables=[\"context\", \"question\", \"generation\", \"error\"],\n",
" )\n",
" \n",
"\n",
" # Chain\n",
" chain = (\n",
" {\n",
" \"context\": lambda x: concatenated_content,\n",
" \"context\": lambda _: concatenated_content,\n",
" \"question\": itemgetter(\"question\"),\n",
" \"generation\": itemgetter(\"generation\"),\n",
" \"error\": itemgetter(\"error\"),\n",
" }\n",
" | prompt\n",
" | llm_with_tool \n",
" | llm_with_tool\n",
" | parser_tool\n",
" )\n",
"\n",
" code_solution = chain.invoke({\"question\":question,\n",
" \"generation\":str(code_solution[0]),\n",
" \"error\":error})\n",
" \n",
" code_solution = chain.invoke(\n",
" {\"question\": question, \"generation\": str(code_solution[0]), \"error\": error}\n",
" )\n",
"\n",
" else:\n",
" print(\"---GENERATE SOLUTION---\")\n",
" \n",
" # Prompt \n",
"\n",
" # Prompt\n",
" prompt = PromptTemplate(\n",
" template=template,\n",
" input_variables=[\"context\", \"question\"],\n",
@@ -266,21 +268,23 @@
" # Chain\n",
" chain = (\n",
" {\n",
" # \"context\": lambda x: docs,\n",
" \"context\": lambda x: concatenated_content,\n",
" \"context\": lambda _: concatenated_content,\n",
" \"question\": itemgetter(\"question\"),\n",
" }\n",
" | prompt\n",
" | llm_with_tool \n",
" | llm_with_tool\n",
" | parser_tool\n",
" )\n",
"\n",
" code_solution = chain.invoke({\"question\":question})\n",
" code_solution = chain.invoke({\"question\": question})\n",
"\n",
" iter = iter+1 \n",
" return {\"keys\": {\"generation\": code_solution, \"question\": question, \"iterations\":iter}}\n",
" iter = iter + 1\n",
" return {\n",
" \"keys\": {\"generation\": code_solution, \"question\": question, \"iterations\": iter}\n",
" }\n",
"\n",
"def check_code_imports(state):\n",
"\n",
"def check_code_imports(state: GraphState):\n",
" \"\"\"\n",
" Check imports\n",
"\n",
@@ -290,7 +294,7 @@
" Returns:\n",
" state (dict): New key added to state, error\n",
" \"\"\"\n",
" \n",
"\n",
" ## State\n",
" print(\"---CHECKING CODE IMPORTS---\")\n",
" state_dict = state[\"keys\"]\n",
@@ -299,7 +303,7 @@
" imports = code_solution[0].imports\n",
" iter = state_dict[\"iterations\"]\n",
"\n",
" try: \n",
" try:\n",
" # Attempt to execute the imports\n",
" exec(imports)\n",
" except Exception as e:\n",
@@ -308,15 +312,23 @@
" error = f\"Execution error: {e}\"\n",
" if \"error\" in state_dict:\n",
" error_prev_runs = state_dict[\"error\"]\n",
" error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error \n",
" error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error\n",
" else:\n",
" print(\"---CODE IMPORT CHECK: SUCCESS---\")\n",
" # No errors occurred\n",
" error = \"None\"\n",
"\n",
" return {\"keys\": {\"generation\": code_solution, \"question\": question, \"error\": error, \"iterations\":iter}}\n",
" return {\n",
" \"keys\": {\n",
" \"generation\": code_solution,\n",
" \"question\": question,\n",
" \"error\": error,\n",
" \"iterations\": iter,\n",
" }\n",
" }\n",
"\n",
"def check_code_execution(state):\n",
"\n",
"def check_code_execution(state: GraphState):\n",
" \"\"\"\n",
" Check code block execution\n",
"\n",
@@ -326,7 +338,7 @@
" Returns:\n",
" state (dict): New key added to state, error\n",
" \"\"\"\n",
" \n",
"\n",
" ## State\n",
" print(\"---CHECKING CODE EXECUTION---\")\n",
" state_dict = state[\"keys\"]\n",
@@ -335,10 +347,10 @@
" prefix = code_solution[0].prefix\n",
" imports = code_solution[0].imports\n",
" code = code_solution[0].code\n",
" code_block = imports +\"\\n\"+ code\n",
" code_block = imports + \"\\n\" + code\n",
" iter = state_dict[\"iterations\"]\n",
"\n",
" try: \n",
" try:\n",
" # Attempt to execute the code block\n",
" exec(code_block)\n",
" except Exception as e:\n",
@@ -347,23 +359,29 @@
" error = f\"Execution error: {e}\"\n",
" if \"error\" in state_dict:\n",
" error_prev_runs = state_dict[\"error\"]\n",
" error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error \n",
" error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error\n",
" else:\n",
" print(\"---CODE BLOCK CHECK: SUCCESS---\")\n",
" # No errors occurred\n",
" error = \"None\"\n",
"\n",
" return {\"keys\": {\"generation\": code_solution, \n",
" \"question\": question, \n",
" \"error\": error, \n",
" \"prefix\":prefix,\n",
" \"imports\":imports,\n",
" \"iterations\":iter,\n",
" \"code\":code}}\n",
" return {\n",
" \"keys\": {\n",
" \"generation\": code_solution,\n",
" \"question\": question,\n",
" \"error\": error,\n",
" \"prefix\": prefix,\n",
" \"imports\": imports,\n",
" \"iterations\": iter,\n",
" \"code\": code,\n",
" }\n",
" }\n",
"\n",
"\n",
"### Edges\n",
"\n",
"def decide_to_check_code_exec(state):\n",
"\n",
"def decide_to_check_code_exec(state: GraphState):\n",
" \"\"\"\n",
" Determines whether to test code execution, or re-try answer generation.\n",
"\n",
@@ -376,8 +394,6 @@
"\n",
" print(\"---DECIDE TO TEST CODE EXECUTION---\")\n",
" state_dict = state[\"keys\"]\n",
" question = state_dict[\"question\"]\n",
" code_solution = state_dict[\"generation\"]\n",
" error = state_dict[\"error\"]\n",
"\n",
" if error == \"None\":\n",
@@ -390,7 +406,8 @@
" print(\"---DECISION: RE-TRY SOLUTION---\")\n",
" return \"generate\"\n",
"\n",
"def decide_to_finish(state):\n",
"\n",
"def decide_to_finish(state: GraphState):\n",
" \"\"\"\n",
" Determines whether to finish (re-try code 3 times.\n",
"\n",
@@ -403,8 +420,6 @@
"\n",
" print(\"---DECIDE TO TEST CODE EXECUTION---\")\n",
" state_dict = state[\"keys\"]\n",
" question = state_dict[\"question\"]\n",
" code_solution = state_dict[\"generation\"]\n",
" error = state_dict[\"error\"]\n",
" iter = state_dict[\"iterations\"]\n",
"\n",
@@ -466,7 +481,7 @@
"source": [
"## Eval\n",
"\n",
"[Here](https://smith.langchain.com/public/ea1f6ca5-de52-4d36-bd7b-fde3faa74a70/d) is a public dataset of LCEL questions. \n",
"[Here](https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d) is a public dataset of LCEL questions. \n",
" \n",
"Let's create a custom LangSmith evaluator [here](https://docs.smith.langchain.com/evaluation/faq/custom-evaluators) to test them with:\n",
"\n",
@@ -474,6 +489,24 @@
"* Our self-corrective coding assistant"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "dcaec0f7",
"metadata": {},
"outputs": [],
"source": [
"import langsmith\n",
"\n",
"client = langsmith.Client()\n",
"\n",
"public_dataset = (\n",
" \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n",
")\n",
"# Clone the dataset to your tenant to use it\n",
"client.clone_public_dataset(public_dataset)"
]
},
{
"cell_type": "markdown",
"id": "86411645-98f8-4d19-889f-c78f3c026380",
@@ -486,20 +519,23 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"id": "d73dfd70-d266-4be7-9fd7-ed1f5cd432c6",
"metadata": {},
"outputs": [],
"source": [
"from langchain_core.runnables import RunnableLambda\n",
"\n",
"\n",
"## Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
"\n",
"\n",
"## LLM\n",
"model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
"\n",
@@ -517,7 +553,7 @@
"\n",
"# Create a prompt template with format instructions and the query\n",
"prompt = PromptTemplate(\n",
" template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" template=\"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is a full set of LCEL documentation: \n",
" \\n ------- \\n\n",
" {context} \n",
@@ -527,14 +563,17 @@
" Structure your answer with a description of the code solution. \\n\n",
" Then list the imports. And finally list the functioning code block. \\n\n",
" Here is the user question: \\n --- --- --- \\n {question}\"\"\",\n",
" input_variables=[\"question\",\"context\"])\n",
" input_variables=[\"question\", \"context\"],\n",
")\n",
"\n",
"\n",
"def parse_answer_to_dict(x):\n",
" return x[0].dict()\n",
"\n",
"\n",
"chain_base_case = (\n",
" {\n",
" \"context\": lambda x: concatenated_content,\n",
" \"context\": lambda _: concatenated_content,\n",
" \"question\": RunnablePassthrough(),\n",
" }\n",
" | prompt\n",
@@ -546,12 +585,13 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"id": "a980b398-ad4c-4b21-8e6d-77ab03130526",
"metadata": {},
"outputs": [],
"source": [
"answer = chain_base_case.invoke(\"How can I write a RAG chain?\")"
"answer = chain_base_case.invoke(\"How can I write a RAG chain?\")\n",
"answer"
]
},
{
@@ -564,7 +604,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 11,
"id": "ebcebf05-d455-4057-bf4c-6cc134bf62c9",
"metadata": {},
"outputs": [],
@@ -572,16 +612,17 @@
"### No LangGraph\n",
"\n",
"import uuid\n",
"from langsmith import Client\n",
"from langchain.smith import RunEvalConfig, run_on_dataset\n",
"from langsmith.evaluation import EvaluationResult, run_evaluator\n",
"from langsmith.schemas import Example, Run\n",
"from typing import Union\n",
"\n",
"@run_evaluator\n",
"from langchain.smith import RunEvalConfig\n",
"from langsmith import Client\n",
"from langsmith.evaluation import EvaluationResult\n",
"from langsmith.schemas import Example, Run\n",
"\n",
"\n",
"def check_import(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs\n",
" imports = model_outputs['imports']\n",
" imports = model_outputs[\"imports\"]\n",
" try:\n",
" exec(imports)\n",
" score = 1\n",
@@ -589,12 +630,12 @@
" score = 0\n",
" return EvaluationResult(key=\"check_import\", score=score)\n",
"\n",
"@run_evaluator\n",
"\n",
"def check_execution(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs\n",
" imports = model_outputs['imports']\n",
" code = model_outputs['code']\n",
" code_to_execute = imports +\"\\n\"+ code\n",
" imports = model_outputs[\"imports\"]\n",
" code = model_outputs[\"code\"]\n",
" code_to_execute = imports + \"\\n\" + code\n",
" try:\n",
" exec(code_to_execute)\n",
" score = 1\n",
@@ -602,12 +643,11 @@
" score = 0\n",
" return EvaluationResult(key=\"check_execution\", score=score)\n",
"\n",
"\n",
"# Config\n",
"evaluation_config = RunEvalConfig(\n",
" custom_evaluators = [check_import,check_execution],\n",
")\n",
"\n",
"client = Client()"
" evaluators=[check_import, check_execution],\n",
")"
]
},
{
@@ -620,9 +660,9 @@
"# Run eval on base chain\n",
"run_id = uuid.uuid4().hex[:4]\n",
"project_name = \"context-stuffing-no-langgraph\"\n",
"client.run_on_dataset(\n",
"results = client.run_on_dataset(\n",
" dataset_name=\"lcel-teacher-eval\",\n",
" llm_or_chain_factory= lambda: (lambda x: x[\"question\"]) | chain_base_case,\n",
" llm_or_chain_factory=lambda: (lambda x: x[\"question\"]) | chain_base_case,\n",
" evaluation=evaluation_config,\n",
" project_name=f\"{run_id}-{project_name}\",\n",
")"
@@ -645,10 +685,10 @@
"source": [
"### LangGraph\n",
"\n",
"@run_evaluator\n",
"\n",
"def check_import(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs[\"keys\"]\n",
" imports = model_outputs['imports']\n",
" imports = model_outputs[\"imports\"]\n",
" try:\n",
" exec(imports)\n",
" score = 1\n",
@@ -656,12 +696,12 @@
" score = 0\n",
" return EvaluationResult(key=\"check_import\", score=score)\n",
"\n",
"@run_evaluator\n",
"\n",
"def check_execution(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs[\"keys\"]\n",
" imports = model_outputs['imports']\n",
" code = model_outputs['code']\n",
" code_to_execute = imports +\"\\n\"+ code\n",
" imports = model_outputs[\"imports\"]\n",
" code = model_outputs[\"code\"]\n",
" code_to_execute = imports + \"\\n\" + code\n",
" try:\n",
" exec(code_to_execute)\n",
" score = 1\n",
@@ -669,18 +709,22 @@
" score = 0\n",
" return EvaluationResult(key=\"check_execution\", score=score)\n",
"\n",
"\n",
"# Config\n",
"evaluation_config = RunEvalConfig(\n",
" custom_evaluators = [check_import,check_execution],\n",
" custom_evaluators=[check_import, check_execution],\n",
")\n",
"\n",
"config = {\"recursion_limit\": 50}\n",
"def model(input):\n",
" return app.invoke({\"keys\":{**input, \"iterations\":0}},config=config)\n",
"\n",
"\n",
"def model(input: dict):\n",
" return app.invoke({\"keys\": {**input, \"iterations\": 0}}, config=config)\n",
"\n",
"\n",
"run_id = uuid.uuid4().hex[:4]\n",
"project_name = \"context-stuffing-with-langgraph\"\n",
"client.run_on_dataset(\n",
"langgraph_results = client.run_on_dataset(\n",
" dataset_name=\"lcel-teacher-eval\",\n",
" llm_or_chain_factory=model,\n",
" evaluation=evaluation_config,\n",
@@ -705,10 +749,14 @@
"metadata": {},
"outputs": [],
"source": [
"langgraph=[\"80db-context-stuffing-with-langgraph\",\n",
"\"060c-context-stuffing-with-langgraph\",\n",
"\"93cd-context-stuffing-with-langgraph\",\n",
"\"60ef-context-stuffing-with-langgraph\"]"
"# You will have to update these to match the tests you ran.\n",
"# The test name can be found at langgraph_results[\"project_name\"]\n",
"langgraph = [\n",
" \"80db-context-stuffing-with-langgraph\",\n",
" \"060c-context-stuffing-with-langgraph\",\n",
" \"93cd-context-stuffing-with-langgraph\",\n",
" \"60ef-context-stuffing-with-langgraph\",\n",
"]"
]
},
{
@@ -718,10 +766,12 @@
"metadata": {},
"outputs": [],
"source": [
"no_langgraph=[\"b493-context-stuffing-no-langgraph\",\n",
"\"eb8a-context-stuffing-no-langgraph\",\n",
"\"b88c-context-stuffing-no-langgraph\",\n",
"\"0aaa-context-stuffing-no-langgraph\"]"
"no_langgraph = [\n",
" \"b493-context-stuffing-no-langgraph\",\n",
" \"eb8a-context-stuffing-no-langgraph\",\n",
" \"b88c-context-stuffing-no-langgraph\",\n",
" \"0aaa-context-stuffing-no-langgraph\",\n",
"]"
]
},
{
@@ -731,19 +781,27 @@
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd \n",
"import pandas as pd\n",
"\n",
"\n",
"def prepare_dataframe(project, trial_number, chain):\n",
" df = client.get_test_results(project_name=project)\n",
" df = df.dropna(subset=['feedback.check_execution', 'feedback.check_import'])\n",
" df = df[['input.question', 'feedback.check_execution', 'feedback.check_import']]\n",
" df['trial #'] = trial_number\n",
" df['chain'] = chain\n",
" df = df.dropna(subset=[\"feedback.check_execution\", \"feedback.check_import\"])\n",
" df = df[[\"input.question\", \"feedback.check_execution\", \"feedback.check_import\"]]\n",
" df[\"trial #\"] = trial_number\n",
" df[\"chain\"] = chain\n",
" return df\n",
"\n",
"\n",
"# Prepare each dataframe\n",
"dfs_chain1 = [prepare_dataframe(project, i+1, 'LangGraph') for i, project in enumerate(langgraph)]\n",
"dfs_chain2 = [prepare_dataframe(project, i+1, 'No LangGraph') for i, project in enumerate(no_langgraph)]\n",
"dfs_chain1 = [\n",
" prepare_dataframe(project, i + 1, \"LangGraph\")\n",
" for i, project in enumerate(langgraph)\n",
"]\n",
"dfs_chain2 = [\n",
" prepare_dataframe(project, i + 1, \"No LangGraph\")\n",
" for i, project in enumerate(no_langgraph)\n",
"]\n",
"\n",
"# Combine all dataframes\n",
"final_df = pd.concat(dfs_chain1 + dfs_chain2, ignore_index=True)"
@@ -851,8 +909,7 @@
],
"source": [
"import pandas as pd\n",
"import seaborn as sns\n",
"import matplotlib.pyplot as plt\n",
"\n",
"\n",
"def group_standard_error(group):\n",
" \"\"\"\n",
@@ -870,9 +927,9 @@
" pd.Series: A series containing the standard error of the 'correct' column.\n",
" \"\"\"\n",
" # 3 trials x 20 questions per trial = 60\n",
" total_trials = len(group) \n",
" total_trials = len(group)\n",
" std_errors = {}\n",
" for column in [\"feedback.check_import\",\"feedback.check_execution\"]:\n",
" for column in [\"feedback.check_import\", \"feedback.check_execution\"]:\n",
" # Number correct\n",
" occurrences = group[column].sum()\n",
" # Total trials\n",
@@ -881,17 +938,30 @@
" std_errors[column] = (fraction * (1 - fraction) / total_trials) ** 0.5\n",
" return pd.Series(std_errors)\n",
"\n",
"\n",
"# Calculate standard errors\n",
"std_errors = final_df.groupby([\"chain\"]).apply(group_standard_error)\n",
"\n",
"# Calculate the fraction of correct answers\n",
"grouped_frac_correct = final_df.groupby('chain')[[\"feedback.check_import\",\"feedback.check_execution\"]].sum() / final_df.groupby('chain')[[\"feedback.check_import\",\"feedback.check_execution\"]].count()\n",
"grouped_frac_correct = (\n",
" final_df.groupby(\"chain\")[\n",
" [\"feedback.check_import\", \"feedback.check_execution\"]\n",
" ].sum()\n",
" / final_df.groupby(\"chain\")[\n",
" [\"feedback.check_import\", \"feedback.check_execution\"]\n",
" ].count()\n",
")\n",
"\n",
"# Concatenate the fraction correct data with the standard errors\n",
"correct_frac_and_errors = pd.concat([grouped_frac_correct, std_errors], axis=1)\n",
"\n",
"# If you want to rename the columns for clarity\n",
"correct_frac_and_errors.columns = [\"Fraction Imports Correct\", \"Fraction Execution Correct\", \"Imports Correct Std Error\", \"Execution Correct Std Error\"]\n",
"correct_frac_and_errors.columns = [\n",
" \"Fraction Imports Correct\",\n",
" \"Fraction Execution Correct\",\n",
" \"Imports Correct Std Error\",\n",
" \"Execution Correct Std Error\",\n",
"]\n",
"correct_frac_and_errors"
]
},
@@ -916,6 +986,7 @@
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"\n",
"\n",
"def plt_combined_bar_graph(df, fraction_fields, error_fields, titles, ylabels):\n",
" \"\"\"\n",
" Plot bar graphs with error bars for specified fields in the provided DataFrame as subplots.\n",
@@ -931,15 +1002,19 @@
" \"\"\"\n",
" n = len(fraction_fields) # Number of plots to create\n",
" fig, axs = plt.subplots(1, n, figsize=(10 * n, 9), sharey=True)\n",
" \n",
" for i, (fraction_field, error_field, title, ylabel) in enumerate(zip(fraction_fields, error_fields, titles, ylabels)):\n",
"\n",
" for i, (fraction_field, error_field, title, ylabel) in enumerate(\n",
" zip(fraction_fields, error_fields, titles, ylabels)\n",
" ):\n",
" barplot = sns.barplot(\n",
" x=\"chain\",\n",
" y=fraction_field,\n",
" data=df.sort_values(\"chain\", ascending=False), # Sort the DataFrame to reverse the order\n",
" data=df.sort_values(\n",
" \"chain\", ascending=False\n",
" ), # Sort the DataFrame to reverse the order\n",
" ax=axs[i],\n",
" capsize=0.1,\n",
" errorbar=None \n",
" errorbar=None,\n",
" )\n",
"\n",
" # Add error bars manually\n",
@@ -951,9 +1026,9 @@
" x=bar.get_x() + bar.get_width() / 2,\n",
" y=bar.get_height(),\n",
" yerr=error,\n",
" fmt='none',\n",
" fmt=\"none\",\n",
" capsize=5,\n",
" color='black'\n",
" color=\"black\",\n",
" )\n",
"\n",
" axs[i].set_title(title)\n",
@@ -963,18 +1038,21 @@
" plt.tight_layout()\n",
" plt.show()\n",
"\n",
"\n",
"# Define the columns and labels for the plots\n",
"fraction_fields = [\"Fraction Imports Correct\", \"Fraction Execution Correct\"]\n",
"error_fields = [\"Imports Correct Std Error\", \"Execution Correct Std Error\"]\n",
"\n",
"titles = [\n",
" \"Feedback Check Import Fraction by Chain\",\n",
" \"Feedback Check Execution Fraction by Chain\"\n",
" \"Feedback Check Execution Fraction by Chain\",\n",
"]\n",
"ylabels = [\"Fraction Correct\", \"Fraction Correct\"]\n",
"\n",
"# Call the function with the specified arguments\n",
"plt_combined_bar_graph(correct_frac_and_errors, fraction_fields, error_fields, titles, ylabels)"
"plt_combined_bar_graph(\n",
" correct_frac_and_errors, fraction_fields, error_fields, titles, ylabels\n",
")"
]
},
{
@@ -1002,7 +1080,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
"version": "3.11.2"
}
},
"nbformat": 4,