mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-08-28 10:49:56 +02:00
Format docs (#752)
This commit is contained in:
William FH
@@ -257,6 +257,7 @@
|
||||
" [RunnableLambda(handle_tool_error)], exception_key=\"error\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def handle_tool_error(state) -> dict:\n",
|
||||
" error = state.get(\"error\")\n",
|
||||
" tool_calls = state[\"messages\"][-1].tool_calls\n",
|
||||
@@ -396,6 +397,7 @@
|
||||
" return \"Error: Query failed. Please rewrite your query and try again.\"\n",
|
||||
" return result\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print(db_query_tool.invoke(\"SELECT * FROM Artist LIMIT 10;\"))"
|
||||
]
|
||||
},
|
||||
@@ -456,8 +458,12 @@
|
||||
"\n",
|
||||
"You will call the appropriate tool to execute the query after running this check.\"\"\"\n",
|
||||
"\n",
|
||||
"query_check_prompt = ChatPromptTemplate.from_messages([(\"system\", query_check_system),(\"placeholder\", \"{messages}\")])\n",
|
||||
"query_check = query_check_prompt | ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools([db_query_tool], tool_choice=\"required\")\n",
|
||||
"query_check_prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [(\"system\", query_check_system), (\"placeholder\", \"{messages}\")]\n",
|
||||
")\n",
|
||||
"query_check = query_check_prompt | ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools(\n",
|
||||
" [db_query_tool], tool_choice=\"required\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"query_check.invoke({\"messages\": [(\"user\", \"SELECT * FROM Artist LIMIT 10;\")]})"
|
||||
]
|
||||
@@ -508,9 +514,11 @@
|
||||
"class State(TypedDict):\n",
|
||||
" messages: Annotated[list[AnyMessage], add_messages]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Define a new graph\n",
|
||||
"workflow = StateGraph(State)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Add a node for the first tool call\n",
|
||||
"def first_tool_call(state: State) -> dict[str, list[AIMessage]]:\n",
|
||||
" return {\n",
|
||||
@@ -520,8 +528,7 @@
|
||||
" tool_calls=[\n",
|
||||
" {\n",
|
||||
" \"name\": \"sql_db_list_tables\",\n",
|
||||
" \"args\": {\n",
|
||||
" },\n",
|
||||
" \"args\": {},\n",
|
||||
" \"id\": \"tool_abcd123\",\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
@@ -529,31 +536,41 @@
|
||||
" ]\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def model_check_query(state: State) -> dict[str, list[AIMessage]]:\n",
|
||||
" \"\"\"\n",
|
||||
" Use this tool to double-check if your query is correct before executing it.\n",
|
||||
" \"\"\"\n",
|
||||
" return {\n",
|
||||
" \"messages\": [\n",
|
||||
" query_check.invoke({\"messages\": [state[\"messages\"][-1]]})\n",
|
||||
" ]\n",
|
||||
" }\n",
|
||||
" return {\"messages\": [query_check.invoke({\"messages\": [state[\"messages\"][-1]]})]}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"workflow.add_node(\"first_tool_call\", first_tool_call)\n",
|
||||
"\n",
|
||||
"# Add nodes for the first two tools\n",
|
||||
"workflow.add_node(\"list_tables_tool\", create_tool_node_with_fallback([list_tables_tool]))\n",
|
||||
"workflow.add_node(\n",
|
||||
" \"list_tables_tool\", create_tool_node_with_fallback([list_tables_tool])\n",
|
||||
")\n",
|
||||
"workflow.add_node(\"get_schema_tool\", create_tool_node_with_fallback([get_schema_tool]))\n",
|
||||
"\n",
|
||||
"# Add a node for a model to choose the relevant tables based on the question and available tables\n",
|
||||
"model_get_schema = ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools([get_schema_tool])\n",
|
||||
"workflow.add_node(\"model_get_schema\", lambda state: {\"messages\": [model_get_schema.invoke(state[\"messages\"])],})\n",
|
||||
"model_get_schema = ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools(\n",
|
||||
" [get_schema_tool]\n",
|
||||
")\n",
|
||||
"workflow.add_node(\n",
|
||||
" \"model_get_schema\",\n",
|
||||
" lambda state: {\n",
|
||||
" \"messages\": [model_get_schema.invoke(state[\"messages\"])],\n",
|
||||
" },\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Describe a tool to represent the end state\n",
|
||||
"class SubmitFinalAnswer(BaseModel):\n",
|
||||
" \"\"\"Submit the final answer to the user based on the query results.\"\"\"\n",
|
||||
"\n",
|
||||
" final_answer: str = Field(..., description=\"The final answer to the user\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Add a node for a model to generate a query based on the question and schema\n",
|
||||
"query_gen_system = \"\"\"You are a SQL expert with a strong attention to detail.\n",
|
||||
"\n",
|
||||
@@ -577,11 +594,17 @@
|
||||
"If you have enough information to answer the input question, simply invoke the appropriate tool to submit the final answer to the user.\n",
|
||||
"\n",
|
||||
"DO NOT make any DML statements (INSERT, UPDATE, DELETE, DROP etc.) to the database.\"\"\"\n",
|
||||
"query_gen_prompt = ChatPromptTemplate.from_messages([(\"system\", query_gen_system),(\"placeholder\", \"{messages}\")])\n",
|
||||
"query_gen = query_gen_prompt | ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools([SubmitFinalAnswer])\n",
|
||||
"query_gen_prompt = ChatPromptTemplate.from_messages(\n",
|
||||
" [(\"system\", query_gen_system), (\"placeholder\", \"{messages}\")]\n",
|
||||
")\n",
|
||||
"query_gen = query_gen_prompt | ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools(\n",
|
||||
" [SubmitFinalAnswer]\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def query_gen_node(state: State):\n",
|
||||
" message = query_gen.invoke(state)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Sometimes, the LLM will hallucinate and call the wrong tool. We need to catch this and return an error message.\n",
|
||||
" tool_messages = []\n",
|
||||
" if message.tool_calls:\n",
|
||||
@@ -597,6 +620,7 @@
|
||||
" tool_messages = []\n",
|
||||
" return {\"messages\": [message] + tool_messages}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"workflow.add_node(\"query_gen\", query_gen_node)\n",
|
||||
"\n",
|
||||
"# Add a node for the model to check the query before executing it\n",
|
||||
@@ -605,6 +629,7 @@
|
||||
"# Add node for executing the query\n",
|
||||
"workflow.add_node(\"execute_query\", create_tool_node_with_fallback([db_query_tool]))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Define a conditional edge to decide whether to continue or end the workflow\n",
|
||||
"def should_continue(state: State) -> Literal[END, \"correct_query\", \"query_gen\"]:\n",
|
||||
" messages = state[\"messages\"]\n",
|
||||
@@ -617,6 +642,7 @@
|
||||
" else:\n",
|
||||
" return \"correct_query\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Specify the edges between the nodes\n",
|
||||
"workflow.set_entry_point(\"first_tool_call\")\n",
|
||||
"workflow.add_edge(\"first_tool_call\", \"list_tables_tool\")\n",
|
||||
@@ -719,9 +745,13 @@
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"messages = app.invoke({\"messages\": [(\"user\", \"Which sales agent made the most in sales in 2009?\")]}) \n",
|
||||
"json_str = messages['messages'][-1].additional_kwargs['tool_calls'][0]['function']['arguments']\n",
|
||||
"json.loads(json_str)['final_answer']"
|
||||
"messages = app.invoke(\n",
|
||||
" {\"messages\": [(\"user\", \"Which sales agent made the most in sales in 2009?\")]}\n",
|
||||
")\n",
|
||||
"json_str = messages[\"messages\"][-1].additional_kwargs[\"tool_calls\"][0][\"function\"][\n",
|
||||
" \"arguments\"\n",
|
||||
"]\n",
|
||||
"json.loads(json_str)[\"final_answer\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -731,7 +761,9 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for event in app.stream({\"messages\": [(\"user\", \"Which sales agent made the most in sales in 2009?\")]}):\n",
|
||||
"for event in app.stream(\n",
|
||||
" {\"messages\": [(\"user\", \"Which sales agent made the most in sales in 2009?\")]}\n",
|
||||
"):\n",
|
||||
" print(event)"
|
||||
]
|
||||
},
|
||||
@@ -774,8 +806,10 @@
|
||||
" \"\"\"Use this for answer evaluation\"\"\"\n",
|
||||
" msg = {\"messages\": (\"user\", example[\"input\"])}\n",
|
||||
" messages = app.invoke(msg)\n",
|
||||
" json_str = messages['messages'][-1].additional_kwargs['tool_calls'][0]['function']['arguments']\n",
|
||||
" response = json.loads(json_str)['final_answer']\n",
|
||||
" json_str = messages[\"messages\"][-1].additional_kwargs[\"tool_calls\"][0][\"function\"][\n",
|
||||
" \"arguments\"\n",
|
||||
" ]\n",
|
||||
" response = json.loads(json_str)[\"final_answer\"]\n",
|
||||
" return {\"response\": response}"
|
||||
]
|
||||
},
|
||||
@@ -792,13 +826,14 @@
|
||||
"# Grade prompt\n",
|
||||
"grade_prompt_answer_accuracy = prompt = hub.pull(\"langchain-ai/rag-answer-vs-reference\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def answer_evaluator(run, example) -> dict:\n",
|
||||
" \"\"\"\n",
|
||||
" A simple evaluator for RAG answer accuracy\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" # Get question, ground truth answer, chain\n",
|
||||
" input_question = example.inputs[\"input\"] \n",
|
||||
" input_question = example.inputs[\"input\"]\n",
|
||||
" reference = example.outputs[\"output\"]\n",
|
||||
" prediction = run.outputs[\"response\"]\n",
|
||||
"\n",
|
||||
@@ -809,9 +844,13 @@
|
||||
" answer_grader = grade_prompt_answer_accuracy | llm\n",
|
||||
"\n",
|
||||
" # Run evaluator\n",
|
||||
" score = answer_grader.invoke({\"question\": input_question,\n",
|
||||
" \"correct_answer\": reference,\n",
|
||||
" \"student_answer\": prediction})\n",
|
||||
" score = answer_grader.invoke(\n",
|
||||
" {\n",
|
||||
" \"question\": input_question,\n",
|
||||
" \"correct_answer\": reference,\n",
|
||||
" \"student_answer\": prediction,\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
" score = score[\"Score\"]\n",
|
||||
"\n",
|
||||
" return {\"key\": \"answer_v_reference_score\", \"score\": score}"
|
||||
@@ -872,10 +911,12 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# These are the tools that we expect the agent to use\n",
|
||||
"expected_trajectory=['sql_db_list_tables', # first: list_tables_tool node\n",
|
||||
" 'sql_db_schema', # second: get_schema_tool node\n",
|
||||
" 'db_query_tool', # third: execute_query node\n",
|
||||
" 'SubmitFinalAnswer'] # fourth: query_gen"
|
||||
"expected_trajectory = [\n",
|
||||
" \"sql_db_list_tables\", # first: list_tables_tool node\n",
|
||||
" \"sql_db_schema\", # second: get_schema_tool node\n",
|
||||
" \"db_query_tool\", # third: execute_query node\n",
|
||||
" \"SubmitFinalAnswer\",\n",
|
||||
"] # fourth: query_gen"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -903,48 +944,59 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"def find_tool_calls(messages):\n",
|
||||
" \"\"\" \n",
|
||||
" Find all tool calls in the messages returned \n",
|
||||
" \"\"\"\n",
|
||||
" tool_calls = [tc['name'] for m in messages['messages'] for tc in getattr(m, 'tool_calls', [])]\n",
|
||||
" Find all tool calls in the messages returned\n",
|
||||
" \"\"\"\n",
|
||||
" tool_calls = [\n",
|
||||
" tc[\"name\"] for m in messages[\"messages\"] for tc in getattr(m, \"tool_calls\", [])\n",
|
||||
" ]\n",
|
||||
" return tool_calls\n",
|
||||
"\n",
|
||||
"def contains_all_tool_calls_in_order_exact_match(root_run: Run, example: Example) -> dict:\n",
|
||||
"\n",
|
||||
"def contains_all_tool_calls_in_order_exact_match(\n",
|
||||
" root_run: Run, example: Example\n",
|
||||
") -> dict:\n",
|
||||
" \"\"\"\n",
|
||||
" Check if all expected tools are called in exact order and without any additional tool calls.\n",
|
||||
" \"\"\"\n",
|
||||
" expected_trajectory = ['sql_db_list_tables', 'sql_db_schema', 'db_query_tool', 'SubmitFinalAnswer']\n",
|
||||
" expected_trajectory = [\n",
|
||||
" \"sql_db_list_tables\",\n",
|
||||
" \"sql_db_schema\",\n",
|
||||
" \"db_query_tool\",\n",
|
||||
" \"SubmitFinalAnswer\",\n",
|
||||
" ]\n",
|
||||
" messages = root_run.outputs[\"response\"]\n",
|
||||
" tool_calls = find_tool_calls(messages)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Print the tool calls for debugging\n",
|
||||
" print(\"Here are my tool calls:\")\n",
|
||||
" print(tool_calls)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Check if the tool calls match the expected trajectory exactly\n",
|
||||
" if tool_calls == expected_trajectory:\n",
|
||||
" score = 1\n",
|
||||
" else:\n",
|
||||
" score = 0\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" return {\"score\": int(score), \"key\": \"multi_tool_call_in_exact_order\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def contains_all_tool_calls_in_order(root_run: Run, example: Example) -> dict:\n",
|
||||
" \"\"\"\n",
|
||||
" Check if all expected tools are called in order, \n",
|
||||
" Check if all expected tools are called in order,\n",
|
||||
" but it allows for other tools to be called in between the expected ones.\n",
|
||||
" \"\"\"\n",
|
||||
" messages = root_run.outputs[\"response\"]\n",
|
||||
" tool_calls = find_tool_calls(messages)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # Print the tool calls for debugging\n",
|
||||
" print(\"Here are my tool calls:\")\n",
|
||||
" print(tool_calls)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" it = iter(tool_calls)\n",
|
||||
" if all(elem in it for elem in expected_trajectory):\n",
|
||||
" score = 1\n",
|
||||
" else: \n",
|
||||
" else:\n",
|
||||
" score = 0\n",
|
||||
" return {\"score\": int(score), \"key\": \"multi_tool_call_in_order\"}"
|
||||
]
|
||||
@@ -959,7 +1011,10 @@
|
||||
"experiment_results = evaluate(\n",
|
||||
" predict_sql_agent_messages,\n",
|
||||
" data=dataset_name,\n",
|
||||
" evaluators=[contains_all_tool_calls_in_order,contains_all_tool_calls_in_order_exact_match],\n",
|
||||
" evaluators=[\n",
|
||||
" contains_all_tool_calls_in_order,\n",
|
||||
" contains_all_tool_calls_in_order_exact_match,\n",
|
||||
" ],\n",
|
||||
" num_repetitions=3,\n",
|
||||
" experiment_prefix=\"sql-agent-multi-step-tool-calling-trajecory-in-order\",\n",
|
||||
" metadata={\"version\": \"Chinook, gpt-4o multi-step-agent\"},\n",
|
||||
|
||||
Reference in New Issue
Block a user