diff --git a/examples/code_assistant/langgraph_code_assistant.ipynb b/examples/code_assistant/langgraph_code_assistant.ipynb index a0183de5e..bc60d25bd 100644 --- a/examples/code_assistant/langgraph_code_assistant.ipynb +++ b/examples/code_assistant/langgraph_code_assistant.ipynb @@ -1,15 +1,5 @@ { "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "10291bbc-be96-4d65-8b57-e39950332a21", - "metadata": {}, - "outputs": [], - "source": [ - "! pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph faiss-cpu" - ] - }, { "attachments": { "15ffc4e1-0f2b-49fb-99c9-2bbda7643172.png": { @@ -51,6 +41,17 @@ "![Screenshot 2024-02-16 at 11.43.52 AM.png](attachment:fb3f0be0-4884-4ad2-b9b3-cf92cfc51273.png)" ] }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e3900420", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture --no-stderr\n", + "%pip install -U langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph faiss-cpu" + ] + }, { "cell_type": "markdown", "id": "38330223-d8c8-4156-82b6-93e63343bc01", @@ -63,7 +64,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "c2eb35d1-4990-47dc-a5c4-208bae588a82", "metadata": {}, "outputs": [], @@ -71,7 +72,7 @@ "from bs4 import BeautifulSoup as Soup\n", "from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n", "\n", - "# LCEL docs \n", + "# LCEL docs\n", "url = \"https://python.langchain.com/docs/expression_language/\"\n", "loader = RecursiveUrlLoader(\n", " url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n", @@ -92,7 +93,7 @@ ")\n", "docs_sq = loader.load()\n", "\n", - "# Add \n", + "# Add\n", "docs.extend([*docs_pydantic, *docs_sq])\n", "\n", "# Sort the list based on the URLs in 'metadata' -> 'source'\n", @@ -117,15 +118,13 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "c185f1a2-e943-4bed-b833-4243c9c64092", "metadata": {}, "outputs": [], "source": [ "from typing import Dict, TypedDict\n", "\n", - "from langchain_core.messages import BaseMessage\n", - "\n", "\n", "class GraphState(TypedDict):\n", " \"\"\"\n", @@ -156,17 +155,19 @@ "outputs": [], "source": [ "from operator import itemgetter\n", - "from langchain_openai import ChatOpenAI\n", - "from langchain.prompts import PromptTemplate\n", - "from langchain_core.runnables import RunnablePassthrough\n", - "from langchain_core.pydantic_v1 import BaseModel, Field\n", - "from langchain.output_parsers.openai_tools import PydanticToolsParser\n", - "from langchain_core.utils.function_calling import convert_to_openai_tool\n", "\n", - "def generate(state):\n", + "from langchain.output_parsers.openai_tools import PydanticToolsParser\n", + "from langchain.prompts import PromptTemplate\n", + "from langchain_core.pydantic_v1 import BaseModel, Field\n", + "from langchain_core.runnables import RunnablePassthrough\n", + "from langchain_core.utils.function_calling import convert_to_openai_tool\n", + "from langchain_openai import ChatOpenAI\n", + "\n", + "\n", + "def generate(state: GraphState):\n", " \"\"\"\n", - " Generate a code solution based on LCEL docs and the input question \n", - " with optional feedback from code execution tests \n", + " Generate a code solution based on LCEL docs and the input question\n", + " with optional feedback from code execution tests\n", "\n", " Args:\n", " state (dict): The current graph state\n", @@ -174,34 +175,35 @@ " Returns:\n", " state (dict): New key added to state, documents, that contains retrieved documents\n", " \"\"\"\n", - " \n", + "\n", " ## State\n", " state_dict = state[\"keys\"]\n", " question = state_dict[\"question\"]\n", " iter = state_dict[\"iterations\"]\n", - " \n", + "\n", " ## Data model\n", " class code(BaseModel):\n", " \"\"\"Code output\"\"\"\n", + "\n", " prefix: str = Field(description=\"Description of the problem and approach\")\n", " imports: str = Field(description=\"Code block import statements\")\n", " code: str = Field(description=\"Code block not including import statements\")\n", - " \n", + "\n", " ## LLM\n", " model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n", - " \n", + "\n", " # Tool\n", " code_tool_oai = convert_to_openai_tool(code)\n", - " \n", + "\n", " # LLM with tool and enforce invocation\n", " llm_with_tool = model.bind(\n", " tools=[code_tool_oai],\n", " tool_choice={\"type\": \"function\", \"function\": {\"name\": \"code\"}},\n", " )\n", - " \n", + "\n", " # Parser\n", " parser_tool = PydanticToolsParser(tools=[code])\n", - " \n", + "\n", " ## Prompt\n", " template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n", " Here is a full set of LCEL documentation: \n", @@ -217,11 +219,11 @@ " ## Generation\n", " if \"error\" in state_dict:\n", " print(\"---RE-GENERATE SOLUTION w/ ERROR FEEDBACK---\")\n", - " \n", + "\n", " error = state_dict[\"error\"]\n", " code_solution = state_dict[\"generation\"]\n", - " \n", - " # Udpate prompt \n", + "\n", + " # Udpate prompt\n", " addendum = \"\"\" \\n --- --- --- \\n You previously tried to solve this problem. \\n Here is your solution: \n", " \\n --- --- --- \\n {generation} \\n --- --- --- \\n Here is the resulting error from code \n", " execution: \\n --- --- --- \\n {error} \\n --- --- --- \\n Please re-try to answer this. \n", @@ -229,35 +231,35 @@ " And finally list the functioning code block. Structure your answer with a description of \n", " the code solution. \\n Then list the imports. And finally list the functioning code block. \n", " \\n Here is the user question: \\n --- --- --- \\n {question}\"\"\"\n", - " template = template + addendum\n", + " template = template + addendum\n", "\n", - " # Prompt \n", + " # Prompt\n", " prompt = PromptTemplate(\n", " template=template,\n", " input_variables=[\"context\", \"question\", \"generation\", \"error\"],\n", " )\n", - " \n", + "\n", " # Chain\n", " chain = (\n", " {\n", - " \"context\": lambda x: concatenated_content,\n", + " \"context\": lambda _: concatenated_content,\n", " \"question\": itemgetter(\"question\"),\n", " \"generation\": itemgetter(\"generation\"),\n", " \"error\": itemgetter(\"error\"),\n", " }\n", " | prompt\n", - " | llm_with_tool \n", + " | llm_with_tool\n", " | parser_tool\n", " )\n", "\n", - " code_solution = chain.invoke({\"question\":question,\n", - " \"generation\":str(code_solution[0]),\n", - " \"error\":error})\n", - " \n", + " code_solution = chain.invoke(\n", + " {\"question\": question, \"generation\": str(code_solution[0]), \"error\": error}\n", + " )\n", + "\n", " else:\n", " print(\"---GENERATE SOLUTION---\")\n", - " \n", - " # Prompt \n", + "\n", + " # Prompt\n", " prompt = PromptTemplate(\n", " template=template,\n", " input_variables=[\"context\", \"question\"],\n", @@ -266,21 +268,23 @@ " # Chain\n", " chain = (\n", " {\n", - " # \"context\": lambda x: docs,\n", - " \"context\": lambda x: concatenated_content,\n", + " \"context\": lambda _: concatenated_content,\n", " \"question\": itemgetter(\"question\"),\n", " }\n", " | prompt\n", - " | llm_with_tool \n", + " | llm_with_tool\n", " | parser_tool\n", " )\n", "\n", - " code_solution = chain.invoke({\"question\":question})\n", + " code_solution = chain.invoke({\"question\": question})\n", "\n", - " iter = iter+1 \n", - " return {\"keys\": {\"generation\": code_solution, \"question\": question, \"iterations\":iter}}\n", + " iter = iter + 1\n", + " return {\n", + " \"keys\": {\"generation\": code_solution, \"question\": question, \"iterations\": iter}\n", + " }\n", "\n", - "def check_code_imports(state):\n", + "\n", + "def check_code_imports(state: GraphState):\n", " \"\"\"\n", " Check imports\n", "\n", @@ -290,7 +294,7 @@ " Returns:\n", " state (dict): New key added to state, error\n", " \"\"\"\n", - " \n", + "\n", " ## State\n", " print(\"---CHECKING CODE IMPORTS---\")\n", " state_dict = state[\"keys\"]\n", @@ -299,7 +303,7 @@ " imports = code_solution[0].imports\n", " iter = state_dict[\"iterations\"]\n", "\n", - " try: \n", + " try:\n", " # Attempt to execute the imports\n", " exec(imports)\n", " except Exception as e:\n", @@ -308,15 +312,23 @@ " error = f\"Execution error: {e}\"\n", " if \"error\" in state_dict:\n", " error_prev_runs = state_dict[\"error\"]\n", - " error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error \n", + " error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error\n", " else:\n", " print(\"---CODE IMPORT CHECK: SUCCESS---\")\n", " # No errors occurred\n", " error = \"None\"\n", "\n", - " return {\"keys\": {\"generation\": code_solution, \"question\": question, \"error\": error, \"iterations\":iter}}\n", + " return {\n", + " \"keys\": {\n", + " \"generation\": code_solution,\n", + " \"question\": question,\n", + " \"error\": error,\n", + " \"iterations\": iter,\n", + " }\n", + " }\n", "\n", - "def check_code_execution(state):\n", + "\n", + "def check_code_execution(state: GraphState):\n", " \"\"\"\n", " Check code block execution\n", "\n", @@ -326,7 +338,7 @@ " Returns:\n", " state (dict): New key added to state, error\n", " \"\"\"\n", - " \n", + "\n", " ## State\n", " print(\"---CHECKING CODE EXECUTION---\")\n", " state_dict = state[\"keys\"]\n", @@ -335,10 +347,10 @@ " prefix = code_solution[0].prefix\n", " imports = code_solution[0].imports\n", " code = code_solution[0].code\n", - " code_block = imports +\"\\n\"+ code\n", + " code_block = imports + \"\\n\" + code\n", " iter = state_dict[\"iterations\"]\n", "\n", - " try: \n", + " try:\n", " # Attempt to execute the code block\n", " exec(code_block)\n", " except Exception as e:\n", @@ -347,23 +359,29 @@ " error = f\"Execution error: {e}\"\n", " if \"error\" in state_dict:\n", " error_prev_runs = state_dict[\"error\"]\n", - " error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error \n", + " error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error\n", " else:\n", " print(\"---CODE BLOCK CHECK: SUCCESS---\")\n", " # No errors occurred\n", " error = \"None\"\n", "\n", - " return {\"keys\": {\"generation\": code_solution, \n", - " \"question\": question, \n", - " \"error\": error, \n", - " \"prefix\":prefix,\n", - " \"imports\":imports,\n", - " \"iterations\":iter,\n", - " \"code\":code}}\n", + " return {\n", + " \"keys\": {\n", + " \"generation\": code_solution,\n", + " \"question\": question,\n", + " \"error\": error,\n", + " \"prefix\": prefix,\n", + " \"imports\": imports,\n", + " \"iterations\": iter,\n", + " \"code\": code,\n", + " }\n", + " }\n", + "\n", "\n", "### Edges\n", "\n", - "def decide_to_check_code_exec(state):\n", + "\n", + "def decide_to_check_code_exec(state: GraphState):\n", " \"\"\"\n", " Determines whether to test code execution, or re-try answer generation.\n", "\n", @@ -376,8 +394,6 @@ "\n", " print(\"---DECIDE TO TEST CODE EXECUTION---\")\n", " state_dict = state[\"keys\"]\n", - " question = state_dict[\"question\"]\n", - " code_solution = state_dict[\"generation\"]\n", " error = state_dict[\"error\"]\n", "\n", " if error == \"None\":\n", @@ -390,7 +406,8 @@ " print(\"---DECISION: RE-TRY SOLUTION---\")\n", " return \"generate\"\n", "\n", - "def decide_to_finish(state):\n", + "\n", + "def decide_to_finish(state: GraphState):\n", " \"\"\"\n", " Determines whether to finish (re-try code 3 times.\n", "\n", @@ -403,8 +420,6 @@ "\n", " print(\"---DECIDE TO TEST CODE EXECUTION---\")\n", " state_dict = state[\"keys\"]\n", - " question = state_dict[\"question\"]\n", - " code_solution = state_dict[\"generation\"]\n", " error = state_dict[\"error\"]\n", " iter = state_dict[\"iterations\"]\n", "\n", @@ -466,7 +481,7 @@ "source": [ "## Eval\n", "\n", - "[Here](https://smith.langchain.com/public/ea1f6ca5-de52-4d36-bd7b-fde3faa74a70/d) is a public dataset of LCEL questions. \n", + "[Here](https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d) is a public dataset of LCEL questions. \n", " \n", "Let's create a custom LangSmith evaluator [here](https://docs.smith.langchain.com/evaluation/faq/custom-evaluators) to test them with:\n", "\n", @@ -474,6 +489,24 @@ "* Our self-corrective coding assistant" ] }, + { + "cell_type": "code", + "execution_count": 8, + "id": "dcaec0f7", + "metadata": {}, + "outputs": [], + "source": [ + "import langsmith\n", + "\n", + "client = langsmith.Client()\n", + "\n", + "public_dataset = (\n", + " \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n", + ")\n", + "# Clone the dataset to your tenant to use it\n", + "client.clone_public_dataset(public_dataset)" + ] + }, { "cell_type": "markdown", "id": "86411645-98f8-4d19-889f-c78f3c026380", @@ -486,20 +519,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "d73dfd70-d266-4be7-9fd7-ed1f5cd432c6", "metadata": {}, "outputs": [], "source": [ "from langchain_core.runnables import RunnableLambda\n", "\n", + "\n", "## Data model\n", "class code(BaseModel):\n", " \"\"\"Code output\"\"\"\n", + "\n", " prefix: str = Field(description=\"Description of the problem and approach\")\n", " imports: str = Field(description=\"Code block import statements\")\n", " code: str = Field(description=\"Code block not including import statements\")\n", "\n", + "\n", "## LLM\n", "model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n", "\n", @@ -517,7 +553,7 @@ "\n", "# Create a prompt template with format instructions and the query\n", "prompt = PromptTemplate(\n", - " template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n", + " template=\"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n", " Here is a full set of LCEL documentation: \n", " \\n ------- \\n\n", " {context} \n", @@ -527,14 +563,17 @@ " Structure your answer with a description of the code solution. \\n\n", " Then list the imports. And finally list the functioning code block. \\n\n", " Here is the user question: \\n --- --- --- \\n {question}\"\"\",\n", - " input_variables=[\"question\",\"context\"])\n", + " input_variables=[\"question\", \"context\"],\n", + ")\n", + "\n", "\n", "def parse_answer_to_dict(x):\n", " return x[0].dict()\n", "\n", + "\n", "chain_base_case = (\n", " {\n", - " \"context\": lambda x: concatenated_content,\n", + " \"context\": lambda _: concatenated_content,\n", " \"question\": RunnablePassthrough(),\n", " }\n", " | prompt\n", @@ -546,12 +585,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "a980b398-ad4c-4b21-8e6d-77ab03130526", "metadata": {}, "outputs": [], "source": [ - "answer = chain_base_case.invoke(\"How can I write a RAG chain?\")" + "answer = chain_base_case.invoke(\"How can I write a RAG chain?\")\n", + "answer" ] }, { @@ -564,7 +604,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "id": "ebcebf05-d455-4057-bf4c-6cc134bf62c9", "metadata": {}, "outputs": [], @@ -572,16 +612,17 @@ "### No LangGraph\n", "\n", "import uuid\n", - "from langsmith import Client\n", - "from langchain.smith import RunEvalConfig, run_on_dataset\n", - "from langsmith.evaluation import EvaluationResult, run_evaluator\n", - "from langsmith.schemas import Example, Run\n", "from typing import Union\n", "\n", - "@run_evaluator\n", + "from langchain.smith import RunEvalConfig\n", + "from langsmith import Client\n", + "from langsmith.evaluation import EvaluationResult\n", + "from langsmith.schemas import Example, Run\n", + "\n", + "\n", "def check_import(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs\n", - " imports = model_outputs['imports']\n", + " imports = model_outputs[\"imports\"]\n", " try:\n", " exec(imports)\n", " score = 1\n", @@ -589,12 +630,12 @@ " score = 0\n", " return EvaluationResult(key=\"check_import\", score=score)\n", "\n", - "@run_evaluator\n", + "\n", "def check_execution(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs\n", - " imports = model_outputs['imports']\n", - " code = model_outputs['code']\n", - " code_to_execute = imports +\"\\n\"+ code\n", + " imports = model_outputs[\"imports\"]\n", + " code = model_outputs[\"code\"]\n", + " code_to_execute = imports + \"\\n\" + code\n", " try:\n", " exec(code_to_execute)\n", " score = 1\n", @@ -602,12 +643,11 @@ " score = 0\n", " return EvaluationResult(key=\"check_execution\", score=score)\n", "\n", + "\n", "# Config\n", "evaluation_config = RunEvalConfig(\n", - " custom_evaluators = [check_import,check_execution],\n", - ")\n", - "\n", - "client = Client()" + " evaluators=[check_import, check_execution],\n", + ")" ] }, { @@ -620,9 +660,9 @@ "# Run eval on base chain\n", "run_id = uuid.uuid4().hex[:4]\n", "project_name = \"context-stuffing-no-langgraph\"\n", - "client.run_on_dataset(\n", + "results = client.run_on_dataset(\n", " dataset_name=\"lcel-teacher-eval\",\n", - " llm_or_chain_factory= lambda: (lambda x: x[\"question\"]) | chain_base_case,\n", + " llm_or_chain_factory=lambda: (lambda x: x[\"question\"]) | chain_base_case,\n", " evaluation=evaluation_config,\n", " project_name=f\"{run_id}-{project_name}\",\n", ")" @@ -645,10 +685,10 @@ "source": [ "### LangGraph\n", "\n", - "@run_evaluator\n", + "\n", "def check_import(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs[\"keys\"]\n", - " imports = model_outputs['imports']\n", + " imports = model_outputs[\"imports\"]\n", " try:\n", " exec(imports)\n", " score = 1\n", @@ -656,12 +696,12 @@ " score = 0\n", " return EvaluationResult(key=\"check_import\", score=score)\n", "\n", - "@run_evaluator\n", + "\n", "def check_execution(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs[\"keys\"]\n", - " imports = model_outputs['imports']\n", - " code = model_outputs['code']\n", - " code_to_execute = imports +\"\\n\"+ code\n", + " imports = model_outputs[\"imports\"]\n", + " code = model_outputs[\"code\"]\n", + " code_to_execute = imports + \"\\n\" + code\n", " try:\n", " exec(code_to_execute)\n", " score = 1\n", @@ -669,18 +709,22 @@ " score = 0\n", " return EvaluationResult(key=\"check_execution\", score=score)\n", "\n", + "\n", "# Config\n", "evaluation_config = RunEvalConfig(\n", - " custom_evaluators = [check_import,check_execution],\n", + " custom_evaluators=[check_import, check_execution],\n", ")\n", "\n", "config = {\"recursion_limit\": 50}\n", - "def model(input):\n", - " return app.invoke({\"keys\":{**input, \"iterations\":0}},config=config)\n", + "\n", + "\n", + "def model(input: dict):\n", + " return app.invoke({\"keys\": {**input, \"iterations\": 0}}, config=config)\n", + "\n", "\n", "run_id = uuid.uuid4().hex[:4]\n", "project_name = \"context-stuffing-with-langgraph\"\n", - "client.run_on_dataset(\n", + "langgraph_results = client.run_on_dataset(\n", " dataset_name=\"lcel-teacher-eval\",\n", " llm_or_chain_factory=model,\n", " evaluation=evaluation_config,\n", @@ -705,10 +749,14 @@ "metadata": {}, "outputs": [], "source": [ - "langgraph=[\"80db-context-stuffing-with-langgraph\",\n", - "\"060c-context-stuffing-with-langgraph\",\n", - "\"93cd-context-stuffing-with-langgraph\",\n", - "\"60ef-context-stuffing-with-langgraph\"]" + "# You will have to update these to match the tests you ran.\n", + "# The test name can be found at langgraph_results[\"project_name\"]\n", + "langgraph = [\n", + " \"80db-context-stuffing-with-langgraph\",\n", + " \"060c-context-stuffing-with-langgraph\",\n", + " \"93cd-context-stuffing-with-langgraph\",\n", + " \"60ef-context-stuffing-with-langgraph\",\n", + "]" ] }, { @@ -718,10 +766,12 @@ "metadata": {}, "outputs": [], "source": [ - "no_langgraph=[\"b493-context-stuffing-no-langgraph\",\n", - "\"eb8a-context-stuffing-no-langgraph\",\n", - "\"b88c-context-stuffing-no-langgraph\",\n", - "\"0aaa-context-stuffing-no-langgraph\"]" + "no_langgraph = [\n", + " \"b493-context-stuffing-no-langgraph\",\n", + " \"eb8a-context-stuffing-no-langgraph\",\n", + " \"b88c-context-stuffing-no-langgraph\",\n", + " \"0aaa-context-stuffing-no-langgraph\",\n", + "]" ] }, { @@ -731,19 +781,27 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd \n", + "import pandas as pd\n", + "\n", "\n", "def prepare_dataframe(project, trial_number, chain):\n", " df = client.get_test_results(project_name=project)\n", - " df = df.dropna(subset=['feedback.check_execution', 'feedback.check_import'])\n", - " df = df[['input.question', 'feedback.check_execution', 'feedback.check_import']]\n", - " df['trial #'] = trial_number\n", - " df['chain'] = chain\n", + " df = df.dropna(subset=[\"feedback.check_execution\", \"feedback.check_import\"])\n", + " df = df[[\"input.question\", \"feedback.check_execution\", \"feedback.check_import\"]]\n", + " df[\"trial #\"] = trial_number\n", + " df[\"chain\"] = chain\n", " return df\n", "\n", + "\n", "# Prepare each dataframe\n", - "dfs_chain1 = [prepare_dataframe(project, i+1, 'LangGraph') for i, project in enumerate(langgraph)]\n", - "dfs_chain2 = [prepare_dataframe(project, i+1, 'No LangGraph') for i, project in enumerate(no_langgraph)]\n", + "dfs_chain1 = [\n", + " prepare_dataframe(project, i + 1, \"LangGraph\")\n", + " for i, project in enumerate(langgraph)\n", + "]\n", + "dfs_chain2 = [\n", + " prepare_dataframe(project, i + 1, \"No LangGraph\")\n", + " for i, project in enumerate(no_langgraph)\n", + "]\n", "\n", "# Combine all dataframes\n", "final_df = pd.concat(dfs_chain1 + dfs_chain2, ignore_index=True)" @@ -851,8 +909,7 @@ ], "source": [ "import pandas as pd\n", - "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", + "\n", "\n", "def group_standard_error(group):\n", " \"\"\"\n", @@ -870,9 +927,9 @@ " pd.Series: A series containing the standard error of the 'correct' column.\n", " \"\"\"\n", " # 3 trials x 20 questions per trial = 60\n", - " total_trials = len(group) \n", + " total_trials = len(group)\n", " std_errors = {}\n", - " for column in [\"feedback.check_import\",\"feedback.check_execution\"]:\n", + " for column in [\"feedback.check_import\", \"feedback.check_execution\"]:\n", " # Number correct\n", " occurrences = group[column].sum()\n", " # Total trials\n", @@ -881,17 +938,30 @@ " std_errors[column] = (fraction * (1 - fraction) / total_trials) ** 0.5\n", " return pd.Series(std_errors)\n", "\n", + "\n", "# Calculate standard errors\n", "std_errors = final_df.groupby([\"chain\"]).apply(group_standard_error)\n", "\n", "# Calculate the fraction of correct answers\n", - "grouped_frac_correct = final_df.groupby('chain')[[\"feedback.check_import\",\"feedback.check_execution\"]].sum() / final_df.groupby('chain')[[\"feedback.check_import\",\"feedback.check_execution\"]].count()\n", + "grouped_frac_correct = (\n", + " final_df.groupby(\"chain\")[\n", + " [\"feedback.check_import\", \"feedback.check_execution\"]\n", + " ].sum()\n", + " / final_df.groupby(\"chain\")[\n", + " [\"feedback.check_import\", \"feedback.check_execution\"]\n", + " ].count()\n", + ")\n", "\n", "# Concatenate the fraction correct data with the standard errors\n", "correct_frac_and_errors = pd.concat([grouped_frac_correct, std_errors], axis=1)\n", "\n", "# If you want to rename the columns for clarity\n", - "correct_frac_and_errors.columns = [\"Fraction Imports Correct\", \"Fraction Execution Correct\", \"Imports Correct Std Error\", \"Execution Correct Std Error\"]\n", + "correct_frac_and_errors.columns = [\n", + " \"Fraction Imports Correct\",\n", + " \"Fraction Execution Correct\",\n", + " \"Imports Correct Std Error\",\n", + " \"Execution Correct Std Error\",\n", + "]\n", "correct_frac_and_errors" ] }, @@ -916,6 +986,7 @@ "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", + "\n", "def plt_combined_bar_graph(df, fraction_fields, error_fields, titles, ylabels):\n", " \"\"\"\n", " Plot bar graphs with error bars for specified fields in the provided DataFrame as subplots.\n", @@ -931,15 +1002,19 @@ " \"\"\"\n", " n = len(fraction_fields) # Number of plots to create\n", " fig, axs = plt.subplots(1, n, figsize=(10 * n, 9), sharey=True)\n", - " \n", - " for i, (fraction_field, error_field, title, ylabel) in enumerate(zip(fraction_fields, error_fields, titles, ylabels)):\n", + "\n", + " for i, (fraction_field, error_field, title, ylabel) in enumerate(\n", + " zip(fraction_fields, error_fields, titles, ylabels)\n", + " ):\n", " barplot = sns.barplot(\n", " x=\"chain\",\n", " y=fraction_field,\n", - " data=df.sort_values(\"chain\", ascending=False), # Sort the DataFrame to reverse the order\n", + " data=df.sort_values(\n", + " \"chain\", ascending=False\n", + " ), # Sort the DataFrame to reverse the order\n", " ax=axs[i],\n", " capsize=0.1,\n", - " errorbar=None \n", + " errorbar=None,\n", " )\n", "\n", " # Add error bars manually\n", @@ -951,9 +1026,9 @@ " x=bar.get_x() + bar.get_width() / 2,\n", " y=bar.get_height(),\n", " yerr=error,\n", - " fmt='none',\n", + " fmt=\"none\",\n", " capsize=5,\n", - " color='black'\n", + " color=\"black\",\n", " )\n", "\n", " axs[i].set_title(title)\n", @@ -963,18 +1038,21 @@ " plt.tight_layout()\n", " plt.show()\n", "\n", + "\n", "# Define the columns and labels for the plots\n", "fraction_fields = [\"Fraction Imports Correct\", \"Fraction Execution Correct\"]\n", "error_fields = [\"Imports Correct Std Error\", \"Execution Correct Std Error\"]\n", "\n", "titles = [\n", " \"Feedback Check Import Fraction by Chain\",\n", - " \"Feedback Check Execution Fraction by Chain\"\n", + " \"Feedback Check Execution Fraction by Chain\",\n", "]\n", "ylabels = [\"Fraction Correct\", \"Fraction Correct\"]\n", "\n", "# Call the function with the specified arguments\n", - "plt_combined_bar_graph(correct_frac_and_errors, fraction_fields, error_fields, titles, ylabels)" + "plt_combined_bar_graph(\n", + " correct_frac_and_errors, fraction_fields, error_fields, titles, ylabels\n", + ")" ] }, { @@ -1002,7 +1080,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.2" } }, "nbformat": 4,