From 54e367a6289825ce9dd93696039273e6e97e14be Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Wed, 6 Mar 2024 17:33:07 -0800 Subject: [PATCH] format --- .../langgraph_code_assistant.ipynb | 240 +++++++++++------- 1 file changed, 154 insertions(+), 86 deletions(-) diff --git a/examples/code_assistant/langgraph_code_assistant.ipynb b/examples/code_assistant/langgraph_code_assistant.ipynb index c52f68253..bc60d25bd 100644 --- a/examples/code_assistant/langgraph_code_assistant.ipynb +++ b/examples/code_assistant/langgraph_code_assistant.ipynb @@ -72,7 +72,7 @@ "from bs4 import BeautifulSoup as Soup\n", "from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n", "\n", - "# LCEL docs \n", + "# LCEL docs\n", "url = \"https://python.langchain.com/docs/expression_language/\"\n", "loader = RecursiveUrlLoader(\n", " url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n", @@ -93,7 +93,7 @@ ")\n", "docs_sq = loader.load()\n", "\n", - "# Add \n", + "# Add\n", "docs.extend([*docs_pydantic, *docs_sq])\n", "\n", "# Sort the list based on the URLs in 'metadata' -> 'source'\n", @@ -166,8 +166,8 @@ "\n", "def generate(state: GraphState):\n", " \"\"\"\n", - " Generate a code solution based on LCEL docs and the input question \n", - " with optional feedback from code execution tests \n", + " Generate a code solution based on LCEL docs and the input question\n", + " with optional feedback from code execution tests\n", "\n", " Args:\n", " state (dict): The current graph state\n", @@ -175,34 +175,35 @@ " Returns:\n", " state (dict): New key added to state, documents, that contains retrieved documents\n", " \"\"\"\n", - " \n", + "\n", " ## State\n", " state_dict = state[\"keys\"]\n", " question = state_dict[\"question\"]\n", " iter = state_dict[\"iterations\"]\n", - " \n", + "\n", " ## Data model\n", " class code(BaseModel):\n", " \"\"\"Code output\"\"\"\n", + "\n", " prefix: str = Field(description=\"Description of the problem and approach\")\n", " imports: str = Field(description=\"Code block import statements\")\n", " code: str = Field(description=\"Code block not including import statements\")\n", - " \n", + "\n", " ## LLM\n", " model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n", - " \n", + "\n", " # Tool\n", " code_tool_oai = convert_to_openai_tool(code)\n", - " \n", + "\n", " # LLM with tool and enforce invocation\n", " llm_with_tool = model.bind(\n", " tools=[code_tool_oai],\n", " tool_choice={\"type\": \"function\", \"function\": {\"name\": \"code\"}},\n", " )\n", - " \n", + "\n", " # Parser\n", " parser_tool = PydanticToolsParser(tools=[code])\n", - " \n", + "\n", " ## Prompt\n", " template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n", " Here is a full set of LCEL documentation: \n", @@ -218,11 +219,11 @@ " ## Generation\n", " if \"error\" in state_dict:\n", " print(\"---RE-GENERATE SOLUTION w/ ERROR FEEDBACK---\")\n", - " \n", + "\n", " error = state_dict[\"error\"]\n", " code_solution = state_dict[\"generation\"]\n", - " \n", - " # Udpate prompt \n", + "\n", + " # Udpate prompt\n", " addendum = \"\"\" \\n --- --- --- \\n You previously tried to solve this problem. \\n Here is your solution: \n", " \\n --- --- --- \\n {generation} \\n --- --- --- \\n Here is the resulting error from code \n", " execution: \\n --- --- --- \\n {error} \\n --- --- --- \\n Please re-try to answer this. \n", @@ -230,14 +231,14 @@ " And finally list the functioning code block. Structure your answer with a description of \n", " the code solution. \\n Then list the imports. And finally list the functioning code block. \n", " \\n Here is the user question: \\n --- --- --- \\n {question}\"\"\"\n", - " template = template + addendum\n", + " template = template + addendum\n", "\n", - " # Prompt \n", + " # Prompt\n", " prompt = PromptTemplate(\n", " template=template,\n", " input_variables=[\"context\", \"question\", \"generation\", \"error\"],\n", " )\n", - " \n", + "\n", " # Chain\n", " chain = (\n", " {\n", @@ -247,18 +248,18 @@ " \"error\": itemgetter(\"error\"),\n", " }\n", " | prompt\n", - " | llm_with_tool \n", + " | llm_with_tool\n", " | parser_tool\n", " )\n", "\n", - " code_solution = chain.invoke({\"question\":question,\n", - " \"generation\":str(code_solution[0]),\n", - " \"error\":error})\n", - " \n", + " code_solution = chain.invoke(\n", + " {\"question\": question, \"generation\": str(code_solution[0]), \"error\": error}\n", + " )\n", + "\n", " else:\n", " print(\"---GENERATE SOLUTION---\")\n", - " \n", - " # Prompt \n", + "\n", + " # Prompt\n", " prompt = PromptTemplate(\n", " template=template,\n", " input_variables=[\"context\", \"question\"],\n", @@ -271,14 +272,17 @@ " \"question\": itemgetter(\"question\"),\n", " }\n", " | prompt\n", - " | llm_with_tool \n", + " | llm_with_tool\n", " | parser_tool\n", " )\n", "\n", - " code_solution = chain.invoke({\"question\":question})\n", + " code_solution = chain.invoke({\"question\": question})\n", + "\n", + " iter = iter + 1\n", + " return {\n", + " \"keys\": {\"generation\": code_solution, \"question\": question, \"iterations\": iter}\n", + " }\n", "\n", - " iter = iter+1 \n", - " return {\"keys\": {\"generation\": code_solution, \"question\": question, \"iterations\":iter}}\n", "\n", "def check_code_imports(state: GraphState):\n", " \"\"\"\n", @@ -290,7 +294,7 @@ " Returns:\n", " state (dict): New key added to state, error\n", " \"\"\"\n", - " \n", + "\n", " ## State\n", " print(\"---CHECKING CODE IMPORTS---\")\n", " state_dict = state[\"keys\"]\n", @@ -299,7 +303,7 @@ " imports = code_solution[0].imports\n", " iter = state_dict[\"iterations\"]\n", "\n", - " try: \n", + " try:\n", " # Attempt to execute the imports\n", " exec(imports)\n", " except Exception as e:\n", @@ -308,13 +312,21 @@ " error = f\"Execution error: {e}\"\n", " if \"error\" in state_dict:\n", " error_prev_runs = state_dict[\"error\"]\n", - " error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error \n", + " error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error\n", " else:\n", " print(\"---CODE IMPORT CHECK: SUCCESS---\")\n", " # No errors occurred\n", " error = \"None\"\n", "\n", - " return {\"keys\": {\"generation\": code_solution, \"question\": question, \"error\": error, \"iterations\":iter}}\n", + " return {\n", + " \"keys\": {\n", + " \"generation\": code_solution,\n", + " \"question\": question,\n", + " \"error\": error,\n", + " \"iterations\": iter,\n", + " }\n", + " }\n", + "\n", "\n", "def check_code_execution(state: GraphState):\n", " \"\"\"\n", @@ -326,7 +338,7 @@ " Returns:\n", " state (dict): New key added to state, error\n", " \"\"\"\n", - " \n", + "\n", " ## State\n", " print(\"---CHECKING CODE EXECUTION---\")\n", " state_dict = state[\"keys\"]\n", @@ -335,10 +347,10 @@ " prefix = code_solution[0].prefix\n", " imports = code_solution[0].imports\n", " code = code_solution[0].code\n", - " code_block = imports +\"\\n\"+ code\n", + " code_block = imports + \"\\n\" + code\n", " iter = state_dict[\"iterations\"]\n", "\n", - " try: \n", + " try:\n", " # Attempt to execute the code block\n", " exec(code_block)\n", " except Exception as e:\n", @@ -347,22 +359,28 @@ " error = f\"Execution error: {e}\"\n", " if \"error\" in state_dict:\n", " error_prev_runs = state_dict[\"error\"]\n", - " error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error \n", + " error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error\n", " else:\n", " print(\"---CODE BLOCK CHECK: SUCCESS---\")\n", " # No errors occurred\n", " error = \"None\"\n", "\n", - " return {\"keys\": {\"generation\": code_solution, \n", - " \"question\": question, \n", - " \"error\": error, \n", - " \"prefix\":prefix,\n", - " \"imports\":imports,\n", - " \"iterations\":iter,\n", - " \"code\":code}}\n", + " return {\n", + " \"keys\": {\n", + " \"generation\": code_solution,\n", + " \"question\": question,\n", + " \"error\": error,\n", + " \"prefix\": prefix,\n", + " \"imports\": imports,\n", + " \"iterations\": iter,\n", + " \"code\": code,\n", + " }\n", + " }\n", + "\n", "\n", "### Edges\n", "\n", + "\n", "def decide_to_check_code_exec(state: GraphState):\n", " \"\"\"\n", " Determines whether to test code execution, or re-try answer generation.\n", @@ -388,6 +406,7 @@ " print(\"---DECISION: RE-TRY SOLUTION---\")\n", " return \"generate\"\n", "\n", + "\n", "def decide_to_finish(state: GraphState):\n", " \"\"\"\n", " Determines whether to finish (re-try code 3 times.\n", @@ -481,7 +500,9 @@ "\n", "client = langsmith.Client()\n", "\n", - "public_dataset = \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n", + "public_dataset = (\n", + " \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n", + ")\n", "# Clone the dataset to your tenant to use it\n", "client.clone_public_dataset(public_dataset)" ] @@ -509,10 +530,12 @@ "## Data model\n", "class code(BaseModel):\n", " \"\"\"Code output\"\"\"\n", + "\n", " prefix: str = Field(description=\"Description of the problem and approach\")\n", " imports: str = Field(description=\"Code block import statements\")\n", " code: str = Field(description=\"Code block not including import statements\")\n", "\n", + "\n", "## LLM\n", "model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n", "\n", @@ -530,7 +553,7 @@ "\n", "# Create a prompt template with format instructions and the query\n", "prompt = PromptTemplate(\n", - " template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n", + " template=\"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n", " Here is a full set of LCEL documentation: \n", " \\n ------- \\n\n", " {context} \n", @@ -540,11 +563,14 @@ " Structure your answer with a description of the code solution. \\n\n", " Then list the imports. And finally list the functioning code block. \\n\n", " Here is the user question: \\n --- --- --- \\n {question}\"\"\",\n", - " input_variables=[\"question\",\"context\"])\n", + " input_variables=[\"question\", \"context\"],\n", + ")\n", + "\n", "\n", "def parse_answer_to_dict(x):\n", " return x[0].dict()\n", "\n", + "\n", "chain_base_case = (\n", " {\n", " \"context\": lambda _: concatenated_content,\n", @@ -596,7 +622,7 @@ "\n", "def check_import(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs\n", - " imports = model_outputs['imports']\n", + " imports = model_outputs[\"imports\"]\n", " try:\n", " exec(imports)\n", " score = 1\n", @@ -604,11 +630,12 @@ " score = 0\n", " return EvaluationResult(key=\"check_import\", score=score)\n", "\n", + "\n", "def check_execution(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs\n", - " imports = model_outputs['imports']\n", - " code = model_outputs['code']\n", - " code_to_execute = imports +\"\\n\"+ code\n", + " imports = model_outputs[\"imports\"]\n", + " code = model_outputs[\"code\"]\n", + " code_to_execute = imports + \"\\n\" + code\n", " try:\n", " exec(code_to_execute)\n", " score = 1\n", @@ -616,10 +643,11 @@ " score = 0\n", " return EvaluationResult(key=\"check_execution\", score=score)\n", "\n", + "\n", "# Config\n", "evaluation_config = RunEvalConfig(\n", - " evaluators = [check_import,check_execution],\n", - ")\n" + " evaluators=[check_import, check_execution],\n", + ")" ] }, { @@ -634,7 +662,7 @@ "project_name = \"context-stuffing-no-langgraph\"\n", "results = client.run_on_dataset(\n", " dataset_name=\"lcel-teacher-eval\",\n", - " llm_or_chain_factory= lambda: (lambda x: x[\"question\"]) | chain_base_case,\n", + " llm_or_chain_factory=lambda: (lambda x: x[\"question\"]) | chain_base_case,\n", " evaluation=evaluation_config,\n", " project_name=f\"{run_id}-{project_name}\",\n", ")" @@ -657,9 +685,10 @@ "source": [ "### LangGraph\n", "\n", + "\n", "def check_import(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs[\"keys\"]\n", - " imports = model_outputs['imports']\n", + " imports = model_outputs[\"imports\"]\n", " try:\n", " exec(imports)\n", " score = 1\n", @@ -667,11 +696,12 @@ " score = 0\n", " return EvaluationResult(key=\"check_import\", score=score)\n", "\n", + "\n", "def check_execution(run: Run, example: Union[Example, None] = None):\n", " model_outputs = run.outputs[\"keys\"]\n", - " imports = model_outputs['imports']\n", - " code = model_outputs['code']\n", - " code_to_execute = imports +\"\\n\"+ code\n", + " imports = model_outputs[\"imports\"]\n", + " code = model_outputs[\"code\"]\n", + " code_to_execute = imports + \"\\n\" + code\n", " try:\n", " exec(code_to_execute)\n", " score = 1\n", @@ -679,14 +709,18 @@ " score = 0\n", " return EvaluationResult(key=\"check_execution\", score=score)\n", "\n", + "\n", "# Config\n", "evaluation_config = RunEvalConfig(\n", - " custom_evaluators = [check_import,check_execution],\n", + " custom_evaluators=[check_import, check_execution],\n", ")\n", "\n", "config = {\"recursion_limit\": 50}\n", + "\n", + "\n", "def model(input: dict):\n", - " return app.invoke({\"keys\":{**input, \"iterations\":0}},config=config)\n", + " return app.invoke({\"keys\": {**input, \"iterations\": 0}}, config=config)\n", + "\n", "\n", "run_id = uuid.uuid4().hex[:4]\n", "project_name = \"context-stuffing-with-langgraph\"\n", @@ -717,10 +751,12 @@ "source": [ "# You will have to update these to match the tests you ran.\n", "# The test name can be found at langgraph_results[\"project_name\"]\n", - "langgraph=[\"80db-context-stuffing-with-langgraph\",\n", - "\"060c-context-stuffing-with-langgraph\",\n", - "\"93cd-context-stuffing-with-langgraph\",\n", - "\"60ef-context-stuffing-with-langgraph\"]" + "langgraph = [\n", + " \"80db-context-stuffing-with-langgraph\",\n", + " \"060c-context-stuffing-with-langgraph\",\n", + " \"93cd-context-stuffing-with-langgraph\",\n", + " \"60ef-context-stuffing-with-langgraph\",\n", + "]" ] }, { @@ -730,10 +766,12 @@ "metadata": {}, "outputs": [], "source": [ - "no_langgraph=[\"b493-context-stuffing-no-langgraph\",\n", - "\"eb8a-context-stuffing-no-langgraph\",\n", - "\"b88c-context-stuffing-no-langgraph\",\n", - "\"0aaa-context-stuffing-no-langgraph\"]" + "no_langgraph = [\n", + " \"b493-context-stuffing-no-langgraph\",\n", + " \"eb8a-context-stuffing-no-langgraph\",\n", + " \"b88c-context-stuffing-no-langgraph\",\n", + " \"0aaa-context-stuffing-no-langgraph\",\n", + "]" ] }, { @@ -743,19 +781,27 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd \n", + "import pandas as pd\n", + "\n", "\n", "def prepare_dataframe(project, trial_number, chain):\n", " df = client.get_test_results(project_name=project)\n", - " df = df.dropna(subset=['feedback.check_execution', 'feedback.check_import'])\n", - " df = df[['input.question', 'feedback.check_execution', 'feedback.check_import']]\n", - " df['trial #'] = trial_number\n", - " df['chain'] = chain\n", + " df = df.dropna(subset=[\"feedback.check_execution\", \"feedback.check_import\"])\n", + " df = df[[\"input.question\", \"feedback.check_execution\", \"feedback.check_import\"]]\n", + " df[\"trial #\"] = trial_number\n", + " df[\"chain\"] = chain\n", " return df\n", "\n", + "\n", "# Prepare each dataframe\n", - "dfs_chain1 = [prepare_dataframe(project, i+1, 'LangGraph') for i, project in enumerate(langgraph)]\n", - "dfs_chain2 = [prepare_dataframe(project, i+1, 'No LangGraph') for i, project in enumerate(no_langgraph)]\n", + "dfs_chain1 = [\n", + " prepare_dataframe(project, i + 1, \"LangGraph\")\n", + " for i, project in enumerate(langgraph)\n", + "]\n", + "dfs_chain2 = [\n", + " prepare_dataframe(project, i + 1, \"No LangGraph\")\n", + " for i, project in enumerate(no_langgraph)\n", + "]\n", "\n", "# Combine all dataframes\n", "final_df = pd.concat(dfs_chain1 + dfs_chain2, ignore_index=True)" @@ -864,6 +910,7 @@ "source": [ "import pandas as pd\n", "\n", + "\n", "def group_standard_error(group):\n", " \"\"\"\n", " Calculate the standard error for the 'correct' column in a given group.\n", @@ -880,9 +927,9 @@ " pd.Series: A series containing the standard error of the 'correct' column.\n", " \"\"\"\n", " # 3 trials x 20 questions per trial = 60\n", - " total_trials = len(group) \n", + " total_trials = len(group)\n", " std_errors = {}\n", - " for column in [\"feedback.check_import\",\"feedback.check_execution\"]:\n", + " for column in [\"feedback.check_import\", \"feedback.check_execution\"]:\n", " # Number correct\n", " occurrences = group[column].sum()\n", " # Total trials\n", @@ -891,17 +938,30 @@ " std_errors[column] = (fraction * (1 - fraction) / total_trials) ** 0.5\n", " return pd.Series(std_errors)\n", "\n", + "\n", "# Calculate standard errors\n", "std_errors = final_df.groupby([\"chain\"]).apply(group_standard_error)\n", "\n", "# Calculate the fraction of correct answers\n", - "grouped_frac_correct = final_df.groupby('chain')[[\"feedback.check_import\",\"feedback.check_execution\"]].sum() / final_df.groupby('chain')[[\"feedback.check_import\",\"feedback.check_execution\"]].count()\n", + "grouped_frac_correct = (\n", + " final_df.groupby(\"chain\")[\n", + " [\"feedback.check_import\", \"feedback.check_execution\"]\n", + " ].sum()\n", + " / final_df.groupby(\"chain\")[\n", + " [\"feedback.check_import\", \"feedback.check_execution\"]\n", + " ].count()\n", + ")\n", "\n", "# Concatenate the fraction correct data with the standard errors\n", "correct_frac_and_errors = pd.concat([grouped_frac_correct, std_errors], axis=1)\n", "\n", "# If you want to rename the columns for clarity\n", - "correct_frac_and_errors.columns = [\"Fraction Imports Correct\", \"Fraction Execution Correct\", \"Imports Correct Std Error\", \"Execution Correct Std Error\"]\n", + "correct_frac_and_errors.columns = [\n", + " \"Fraction Imports Correct\",\n", + " \"Fraction Execution Correct\",\n", + " \"Imports Correct Std Error\",\n", + " \"Execution Correct Std Error\",\n", + "]\n", "correct_frac_and_errors" ] }, @@ -926,6 +986,7 @@ "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", + "\n", "def plt_combined_bar_graph(df, fraction_fields, error_fields, titles, ylabels):\n", " \"\"\"\n", " Plot bar graphs with error bars for specified fields in the provided DataFrame as subplots.\n", @@ -941,15 +1002,19 @@ " \"\"\"\n", " n = len(fraction_fields) # Number of plots to create\n", " fig, axs = plt.subplots(1, n, figsize=(10 * n, 9), sharey=True)\n", - " \n", - " for i, (fraction_field, error_field, title, ylabel) in enumerate(zip(fraction_fields, error_fields, titles, ylabels)):\n", + "\n", + " for i, (fraction_field, error_field, title, ylabel) in enumerate(\n", + " zip(fraction_fields, error_fields, titles, ylabels)\n", + " ):\n", " barplot = sns.barplot(\n", " x=\"chain\",\n", " y=fraction_field,\n", - " data=df.sort_values(\"chain\", ascending=False), # Sort the DataFrame to reverse the order\n", + " data=df.sort_values(\n", + " \"chain\", ascending=False\n", + " ), # Sort the DataFrame to reverse the order\n", " ax=axs[i],\n", " capsize=0.1,\n", - " errorbar=None \n", + " errorbar=None,\n", " )\n", "\n", " # Add error bars manually\n", @@ -961,9 +1026,9 @@ " x=bar.get_x() + bar.get_width() / 2,\n", " y=bar.get_height(),\n", " yerr=error,\n", - " fmt='none',\n", + " fmt=\"none\",\n", " capsize=5,\n", - " color='black'\n", + " color=\"black\",\n", " )\n", "\n", " axs[i].set_title(title)\n", @@ -973,18 +1038,21 @@ " plt.tight_layout()\n", " plt.show()\n", "\n", + "\n", "# Define the columns and labels for the plots\n", "fraction_fields = [\"Fraction Imports Correct\", \"Fraction Execution Correct\"]\n", "error_fields = [\"Imports Correct Std Error\", \"Execution Correct Std Error\"]\n", "\n", "titles = [\n", " \"Feedback Check Import Fraction by Chain\",\n", - " \"Feedback Check Execution Fraction by Chain\"\n", + " \"Feedback Check Execution Fraction by Chain\",\n", "]\n", "ylabels = [\"Fraction Correct\", \"Fraction Correct\"]\n", "\n", "# Call the function with the specified arguments\n", - "plt_combined_bar_graph(correct_frac_and_errors, fraction_fields, error_fields, titles, ylabels)" + "plt_combined_bar_graph(\n", + " correct_frac_and_errors, fraction_fields, error_fields, titles, ylabels\n", + ")" ] }, {