WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-08-28 10:49:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

format

Browse Source
This commit is contained in:
William Fu-Hinthorn
2024-03-06 17:33:07 -08:00
parent 7e4abdf08a
commit 54e367a628
1 changed files with 154 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/code_assistant/langgraph_code_assistant.ipynb
+154 -86
View File
@@ -72,7 +72,7 @@
"from bs4 import BeautifulSoup as Soup\n",
"from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n",
"\n",
"# LCEL docs \n",
"# LCEL docs\n",
"url = \"https://python.langchain.com/docs/expression_language/\"\n",
"loader = RecursiveUrlLoader(\n",
" url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n",
@@ -93,7 +93,7 @@
")\n",
"docs_sq = loader.load()\n",
"\n",
"# Add \n",
"# Add\n",
"docs.extend([*docs_pydantic, *docs_sq])\n",
"\n",
"# Sort the list based on the URLs in 'metadata' -> 'source'\n",
@@ -166,8 +166,8 @@
"\n",
"def generate(state: GraphState):\n",
" \"\"\"\n",
" Generate a code solution based on LCEL docs and the input question \n",
" with optional feedback from code execution tests \n",
" Generate a code solution based on LCEL docs and the input question\n",
" with optional feedback from code execution tests\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
@@ -175,34 +175,35 @@
" Returns:\n",
" state (dict): New key added to state, documents, that contains retrieved documents\n",
" \"\"\"\n",
" \n",
"\n",
" ## State\n",
" state_dict = state[\"keys\"]\n",
" question = state_dict[\"question\"]\n",
" iter = state_dict[\"iterations\"]\n",
" \n",
"\n",
" ## Data model\n",
" class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
" \n",
"\n",
" ## LLM\n",
" model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
" \n",
"\n",
" # Tool\n",
" code_tool_oai = convert_to_openai_tool(code)\n",
" \n",
"\n",
" # LLM with tool and enforce invocation\n",
" llm_with_tool = model.bind(\n",
" tools=[code_tool_oai],\n",
" tool_choice={\"type\": \"function\", \"function\": {\"name\": \"code\"}},\n",
" )\n",
" \n",
"\n",
" # Parser\n",
" parser_tool = PydanticToolsParser(tools=[code])\n",
" \n",
"\n",
" ## Prompt\n",
" template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is a full set of LCEL documentation: \n",
@@ -218,11 +219,11 @@
" ## Generation\n",
" if \"error\" in state_dict:\n",
" print(\"---RE-GENERATE SOLUTION w/ ERROR FEEDBACK---\")\n",
" \n",
"\n",
" error = state_dict[\"error\"]\n",
" code_solution = state_dict[\"generation\"]\n",
" \n",
" # Udpate prompt \n",
"\n",
" # Udpate prompt\n",
" addendum = \"\"\" \\n --- --- --- \\n You previously tried to solve this problem. \\n Here is your solution: \n",
" \\n --- --- --- \\n {generation} \\n --- --- --- \\n Here is the resulting error from code \n",
" execution: \\n --- --- --- \\n {error} \\n --- --- --- \\n Please re-try to answer this. \n",
@@ -230,14 +231,14 @@
" And finally list the functioning code block. Structure your answer with a description of \n",
" the code solution. \\n Then list the imports. And finally list the functioning code block. \n",
" \\n Here is the user question: \\n --- --- --- \\n {question}\"\"\"\n",
" template = template + addendum\n",
" template = template + addendum\n",
"\n",
" # Prompt \n",
" # Prompt\n",
" prompt = PromptTemplate(\n",
" template=template,\n",
" input_variables=[\"context\", \"question\", \"generation\", \"error\"],\n",
" )\n",
" \n",
"\n",
" # Chain\n",
" chain = (\n",
" {\n",
@@ -247,18 +248,18 @@
" \"error\": itemgetter(\"error\"),\n",
" }\n",
" | prompt\n",
" | llm_with_tool \n",
" | llm_with_tool\n",
" | parser_tool\n",
" )\n",
"\n",
" code_solution = chain.invoke({\"question\":question,\n",
" \"generation\":str(code_solution[0]),\n",
" \"error\":error})\n",
" \n",
" code_solution = chain.invoke(\n",
" {\"question\": question, \"generation\": str(code_solution[0]), \"error\": error}\n",
" )\n",
"\n",
" else:\n",
" print(\"---GENERATE SOLUTION---\")\n",
" \n",
" # Prompt \n",
"\n",
" # Prompt\n",
" prompt = PromptTemplate(\n",
" template=template,\n",
" input_variables=[\"context\", \"question\"],\n",
@@ -271,14 +272,17 @@
" \"question\": itemgetter(\"question\"),\n",
" }\n",
" | prompt\n",
" | llm_with_tool \n",
" | llm_with_tool\n",
" | parser_tool\n",
" )\n",
"\n",
" code_solution = chain.invoke({\"question\":question})\n",
" code_solution = chain.invoke({\"question\": question})\n",
"\n",
" iter = iter + 1\n",
" return {\n",
" \"keys\": {\"generation\": code_solution, \"question\": question, \"iterations\": iter}\n",
" }\n",
"\n",
" iter = iter+1 \n",
" return {\"keys\": {\"generation\": code_solution, \"question\": question, \"iterations\":iter}}\n",
"\n",
"def check_code_imports(state: GraphState):\n",
" \"\"\"\n",
@@ -290,7 +294,7 @@
" Returns:\n",
" state (dict): New key added to state, error\n",
" \"\"\"\n",
" \n",
"\n",
" ## State\n",
" print(\"---CHECKING CODE IMPORTS---\")\n",
" state_dict = state[\"keys\"]\n",
@@ -299,7 +303,7 @@
" imports = code_solution[0].imports\n",
" iter = state_dict[\"iterations\"]\n",
"\n",
" try: \n",
" try:\n",
" # Attempt to execute the imports\n",
" exec(imports)\n",
" except Exception as e:\n",
@@ -308,13 +312,21 @@
" error = f\"Execution error: {e}\"\n",
" if \"error\" in state_dict:\n",
" error_prev_runs = state_dict[\"error\"]\n",
" error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error \n",
" error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error\n",
" else:\n",
" print(\"---CODE IMPORT CHECK: SUCCESS---\")\n",
" # No errors occurred\n",
" error = \"None\"\n",
"\n",
" return {\"keys\": {\"generation\": code_solution, \"question\": question, \"error\": error, \"iterations\":iter}}\n",
" return {\n",
" \"keys\": {\n",
" \"generation\": code_solution,\n",
" \"question\": question,\n",
" \"error\": error,\n",
" \"iterations\": iter,\n",
" }\n",
" }\n",
"\n",
"\n",
"def check_code_execution(state: GraphState):\n",
" \"\"\"\n",
@@ -326,7 +338,7 @@
" Returns:\n",
" state (dict): New key added to state, error\n",
" \"\"\"\n",
" \n",
"\n",
" ## State\n",
" print(\"---CHECKING CODE EXECUTION---\")\n",
" state_dict = state[\"keys\"]\n",
@@ -335,10 +347,10 @@
" prefix = code_solution[0].prefix\n",
" imports = code_solution[0].imports\n",
" code = code_solution[0].code\n",
" code_block = imports +\"\\n\"+ code\n",
" code_block = imports + \"\\n\" + code\n",
" iter = state_dict[\"iterations\"]\n",
"\n",
" try: \n",
" try:\n",
" # Attempt to execute the code block\n",
" exec(code_block)\n",
" except Exception as e:\n",
@@ -347,22 +359,28 @@
" error = f\"Execution error: {e}\"\n",
" if \"error\" in state_dict:\n",
" error_prev_runs = state_dict[\"error\"]\n",
" error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error \n",
" error = error_prev_runs + \"\\n --- Most recent run error --- \\n\" + error\n",
" else:\n",
" print(\"---CODE BLOCK CHECK: SUCCESS---\")\n",
" # No errors occurred\n",
" error = \"None\"\n",
"\n",
" return {\"keys\": {\"generation\": code_solution, \n",
" \"question\": question, \n",
" \"error\": error, \n",
" \"prefix\":prefix,\n",
" \"imports\":imports,\n",
" \"iterations\":iter,\n",
" \"code\":code}}\n",
" return {\n",
" \"keys\": {\n",
" \"generation\": code_solution,\n",
" \"question\": question,\n",
" \"error\": error,\n",
" \"prefix\": prefix,\n",
" \"imports\": imports,\n",
" \"iterations\": iter,\n",
" \"code\": code,\n",
" }\n",
" }\n",
"\n",
"\n",
"### Edges\n",
"\n",
"\n",
"def decide_to_check_code_exec(state: GraphState):\n",
" \"\"\"\n",
" Determines whether to test code execution, or re-try answer generation.\n",
@@ -388,6 +406,7 @@
" print(\"---DECISION: RE-TRY SOLUTION---\")\n",
" return \"generate\"\n",
"\n",
"\n",
"def decide_to_finish(state: GraphState):\n",
" \"\"\"\n",
" Determines whether to finish (re-try code 3 times.\n",
@@ -481,7 +500,9 @@
"\n",
"client = langsmith.Client()\n",
"\n",
"public_dataset = \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n",
"public_dataset = (\n",
" \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n",
")\n",
"# Clone the dataset to your tenant to use it\n",
"client.clone_public_dataset(public_dataset)"
]
@@ -509,10 +530,12 @@
"## Data model\n",
"class code(BaseModel):\n",
" \"\"\"Code output\"\"\"\n",
"\n",
" prefix: str = Field(description=\"Description of the problem and approach\")\n",
" imports: str = Field(description=\"Code block import statements\")\n",
" code: str = Field(description=\"Code block not including import statements\")\n",
"\n",
"\n",
"## LLM\n",
"model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
"\n",
@@ -530,7 +553,7 @@
"\n",
"# Create a prompt template with format instructions and the query\n",
"prompt = PromptTemplate(\n",
" template = \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" template=\"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
" Here is a full set of LCEL documentation: \n",
" \\n ------- \\n\n",
" {context} \n",
@@ -540,11 +563,14 @@
" Structure your answer with a description of the code solution. \\n\n",
" Then list the imports. And finally list the functioning code block. \\n\n",
" Here is the user question: \\n --- --- --- \\n {question}\"\"\",\n",
" input_variables=[\"question\",\"context\"])\n",
" input_variables=[\"question\", \"context\"],\n",
")\n",
"\n",
"\n",
"def parse_answer_to_dict(x):\n",
" return x[0].dict()\n",
"\n",
"\n",
"chain_base_case = (\n",
" {\n",
" \"context\": lambda _: concatenated_content,\n",
@@ -596,7 +622,7 @@
"\n",
"def check_import(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs\n",
" imports = model_outputs['imports']\n",
" imports = model_outputs[\"imports\"]\n",
" try:\n",
" exec(imports)\n",
" score = 1\n",
@@ -604,11 +630,12 @@
" score = 0\n",
" return EvaluationResult(key=\"check_import\", score=score)\n",
"\n",
"\n",
"def check_execution(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs\n",
" imports = model_outputs['imports']\n",
" code = model_outputs['code']\n",
" code_to_execute = imports +\"\\n\"+ code\n",
" imports = model_outputs[\"imports\"]\n",
" code = model_outputs[\"code\"]\n",
" code_to_execute = imports + \"\\n\" + code\n",
" try:\n",
" exec(code_to_execute)\n",
" score = 1\n",
@@ -616,10 +643,11 @@
" score = 0\n",
" return EvaluationResult(key=\"check_execution\", score=score)\n",
"\n",
"\n",
"# Config\n",
"evaluation_config = RunEvalConfig(\n",
" evaluators = [check_import,check_execution],\n",
")\n"
" evaluators=[check_import, check_execution],\n",
")"
]
},
{
@@ -634,7 +662,7 @@
"project_name = \"context-stuffing-no-langgraph\"\n",
"results = client.run_on_dataset(\n",
" dataset_name=\"lcel-teacher-eval\",\n",
" llm_or_chain_factory= lambda: (lambda x: x[\"question\"]) | chain_base_case,\n",
" llm_or_chain_factory=lambda: (lambda x: x[\"question\"]) | chain_base_case,\n",
" evaluation=evaluation_config,\n",
" project_name=f\"{run_id}-{project_name}\",\n",
")"
@@ -657,9 +685,10 @@
"source": [
"### LangGraph\n",
"\n",
"\n",
"def check_import(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs[\"keys\"]\n",
" imports = model_outputs['imports']\n",
" imports = model_outputs[\"imports\"]\n",
" try:\n",
" exec(imports)\n",
" score = 1\n",
@@ -667,11 +696,12 @@
" score = 0\n",
" return EvaluationResult(key=\"check_import\", score=score)\n",
"\n",
"\n",
"def check_execution(run: Run, example: Union[Example, None] = None):\n",
" model_outputs = run.outputs[\"keys\"]\n",
" imports = model_outputs['imports']\n",
" code = model_outputs['code']\n",
" code_to_execute = imports +\"\\n\"+ code\n",
" imports = model_outputs[\"imports\"]\n",
" code = model_outputs[\"code\"]\n",
" code_to_execute = imports + \"\\n\" + code\n",
" try:\n",
" exec(code_to_execute)\n",
" score = 1\n",
@@ -679,14 +709,18 @@
" score = 0\n",
" return EvaluationResult(key=\"check_execution\", score=score)\n",
"\n",
"\n",
"# Config\n",
"evaluation_config = RunEvalConfig(\n",
" custom_evaluators = [check_import,check_execution],\n",
" custom_evaluators=[check_import, check_execution],\n",
")\n",
"\n",
"config = {\"recursion_limit\": 50}\n",
"\n",
"\n",
"def model(input: dict):\n",
" return app.invoke({\"keys\":{**input, \"iterations\":0}},config=config)\n",
" return app.invoke({\"keys\": {**input, \"iterations\": 0}}, config=config)\n",
"\n",
"\n",
"run_id = uuid.uuid4().hex[:4]\n",
"project_name = \"context-stuffing-with-langgraph\"\n",
@@ -717,10 +751,12 @@
"source": [
"# You will have to update these to match the tests you ran.\n",
"# The test name can be found at langgraph_results[\"project_name\"]\n",
"langgraph=[\"80db-context-stuffing-with-langgraph\",\n",
"\"060c-context-stuffing-with-langgraph\",\n",
"\"93cd-context-stuffing-with-langgraph\",\n",
"\"60ef-context-stuffing-with-langgraph\"]"
"langgraph = [\n",
" \"80db-context-stuffing-with-langgraph\",\n",
" \"060c-context-stuffing-with-langgraph\",\n",
" \"93cd-context-stuffing-with-langgraph\",\n",
" \"60ef-context-stuffing-with-langgraph\",\n",
"]"
]
},
{
@@ -730,10 +766,12 @@
"metadata": {},
"outputs": [],
"source": [
"no_langgraph=[\"b493-context-stuffing-no-langgraph\",\n",
"\"eb8a-context-stuffing-no-langgraph\",\n",
"\"b88c-context-stuffing-no-langgraph\",\n",
"\"0aaa-context-stuffing-no-langgraph\"]"
"no_langgraph = [\n",
" \"b493-context-stuffing-no-langgraph\",\n",
" \"eb8a-context-stuffing-no-langgraph\",\n",
" \"b88c-context-stuffing-no-langgraph\",\n",
" \"0aaa-context-stuffing-no-langgraph\",\n",
"]"
]
},
{
@@ -743,19 +781,27 @@
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd \n",
"import pandas as pd\n",
"\n",
"\n",
"def prepare_dataframe(project, trial_number, chain):\n",
" df = client.get_test_results(project_name=project)\n",
" df = df.dropna(subset=['feedback.check_execution', 'feedback.check_import'])\n",
" df = df[['input.question', 'feedback.check_execution', 'feedback.check_import']]\n",
" df['trial #'] = trial_number\n",
" df['chain'] = chain\n",
" df = df.dropna(subset=[\"feedback.check_execution\", \"feedback.check_import\"])\n",
" df = df[[\"input.question\", \"feedback.check_execution\", \"feedback.check_import\"]]\n",
" df[\"trial #\"] = trial_number\n",
" df[\"chain\"] = chain\n",
" return df\n",
"\n",
"\n",
"# Prepare each dataframe\n",
"dfs_chain1 = [prepare_dataframe(project, i+1, 'LangGraph') for i, project in enumerate(langgraph)]\n",
"dfs_chain2 = [prepare_dataframe(project, i+1, 'No LangGraph') for i, project in enumerate(no_langgraph)]\n",
"dfs_chain1 = [\n",
" prepare_dataframe(project, i + 1, \"LangGraph\")\n",
" for i, project in enumerate(langgraph)\n",
"]\n",
"dfs_chain2 = [\n",
" prepare_dataframe(project, i + 1, \"No LangGraph\")\n",
" for i, project in enumerate(no_langgraph)\n",
"]\n",
"\n",
"# Combine all dataframes\n",
"final_df = pd.concat(dfs_chain1 + dfs_chain2, ignore_index=True)"
@@ -864,6 +910,7 @@
"source": [
"import pandas as pd\n",
"\n",
"\n",
"def group_standard_error(group):\n",
" \"\"\"\n",
" Calculate the standard error for the 'correct' column in a given group.\n",
@@ -880,9 +927,9 @@
" pd.Series: A series containing the standard error of the 'correct' column.\n",
" \"\"\"\n",
" # 3 trials x 20 questions per trial = 60\n",
" total_trials = len(group) \n",
" total_trials = len(group)\n",
" std_errors = {}\n",
" for column in [\"feedback.check_import\",\"feedback.check_execution\"]:\n",
" for column in [\"feedback.check_import\", \"feedback.check_execution\"]:\n",
" # Number correct\n",
" occurrences = group[column].sum()\n",
" # Total trials\n",
@@ -891,17 +938,30 @@
" std_errors[column] = (fraction * (1 - fraction) / total_trials) ** 0.5\n",
" return pd.Series(std_errors)\n",
"\n",
"\n",
"# Calculate standard errors\n",
"std_errors = final_df.groupby([\"chain\"]).apply(group_standard_error)\n",
"\n",
"# Calculate the fraction of correct answers\n",
"grouped_frac_correct = final_df.groupby('chain')[[\"feedback.check_import\",\"feedback.check_execution\"]].sum() / final_df.groupby('chain')[[\"feedback.check_import\",\"feedback.check_execution\"]].count()\n",
"grouped_frac_correct = (\n",
" final_df.groupby(\"chain\")[\n",
" [\"feedback.check_import\", \"feedback.check_execution\"]\n",
" ].sum()\n",
" / final_df.groupby(\"chain\")[\n",
" [\"feedback.check_import\", \"feedback.check_execution\"]\n",
" ].count()\n",
")\n",
"\n",
"# Concatenate the fraction correct data with the standard errors\n",
"correct_frac_and_errors = pd.concat([grouped_frac_correct, std_errors], axis=1)\n",
"\n",
"# If you want to rename the columns for clarity\n",
"correct_frac_and_errors.columns = [\"Fraction Imports Correct\", \"Fraction Execution Correct\", \"Imports Correct Std Error\", \"Execution Correct Std Error\"]\n",
"correct_frac_and_errors.columns = [\n",
" \"Fraction Imports Correct\",\n",
" \"Fraction Execution Correct\",\n",
" \"Imports Correct Std Error\",\n",
" \"Execution Correct Std Error\",\n",
"]\n",
"correct_frac_and_errors"
]
},
@@ -926,6 +986,7 @@
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"\n",
"\n",
"def plt_combined_bar_graph(df, fraction_fields, error_fields, titles, ylabels):\n",
" \"\"\"\n",
" Plot bar graphs with error bars for specified fields in the provided DataFrame as subplots.\n",
@@ -941,15 +1002,19 @@
" \"\"\"\n",
" n = len(fraction_fields) # Number of plots to create\n",
" fig, axs = plt.subplots(1, n, figsize=(10 * n, 9), sharey=True)\n",
" \n",
" for i, (fraction_field, error_field, title, ylabel) in enumerate(zip(fraction_fields, error_fields, titles, ylabels)):\n",
"\n",
" for i, (fraction_field, error_field, title, ylabel) in enumerate(\n",
" zip(fraction_fields, error_fields, titles, ylabels)\n",
" ):\n",
" barplot = sns.barplot(\n",
" x=\"chain\",\n",
" y=fraction_field,\n",
" data=df.sort_values(\"chain\", ascending=False), # Sort the DataFrame to reverse the order\n",
" data=df.sort_values(\n",
" \"chain\", ascending=False\n",
" ), # Sort the DataFrame to reverse the order\n",
" ax=axs[i],\n",
" capsize=0.1,\n",
" errorbar=None \n",
" errorbar=None,\n",
" )\n",
"\n",
" # Add error bars manually\n",
@@ -961,9 +1026,9 @@
" x=bar.get_x() + bar.get_width() / 2,\n",
" y=bar.get_height(),\n",
" yerr=error,\n",
" fmt='none',\n",
" fmt=\"none\",\n",
" capsize=5,\n",
" color='black'\n",
" color=\"black\",\n",
" )\n",
"\n",
" axs[i].set_title(title)\n",
@@ -973,18 +1038,21 @@
" plt.tight_layout()\n",
" plt.show()\n",
"\n",
"\n",
"# Define the columns and labels for the plots\n",
"fraction_fields = [\"Fraction Imports Correct\", \"Fraction Execution Correct\"]\n",
"error_fields = [\"Imports Correct Std Error\", \"Execution Correct Std Error\"]\n",
"\n",
"titles = [\n",
" \"Feedback Check Import Fraction by Chain\",\n",
" \"Feedback Check Execution Fraction by Chain\"\n",
" \"Feedback Check Execution Fraction by Chain\",\n",
"]\n",
"ylabels = [\"Fraction Correct\", \"Fraction Correct\"]\n",
"\n",
"# Call the function with the specified arguments\n",
"plt_combined_bar_graph(correct_frac_and_errors, fraction_fields, error_fields, titles, ylabels)"
"plt_combined_bar_graph(\n",
" correct_frac_and_errors, fraction_fields, error_fields, titles, ylabels\n",
")"
]
},
{