langgraph: add message list validation to create_react_agent + a troubleshooting guide (#2182)

docs/docs/how-tos/index.md

@@ -220,11 +220,12 @@ LangGraph Studio is a built-in UI for visualizing, testing, and debugging your a
 
 ## Troubleshooting
 
-The [Error Reference](../troubleshooting/errors/index.md) page contains guides around resolving common errors you may find while building with LangGraph. Errors referenced below will have an `lc_error_code` property corresponding to one of the below codes when they are thrown in code.
+These are the guides for resolving common errors you may find while building with LangGraph. Errors referenced below will have an `lc_error_code` property corresponding to one of the below codes when they are thrown in code.
 
 - [GRAPH_RECURSION_LIMIT](../troubleshooting/errors/GRAPH_RECURSION_LIMIT.md)
 - [INVALID_CONCURRENT_GRAPH_UPDATE](../troubleshooting/errors/INVALID_CONCURRENT_GRAPH_UPDATE.md)
 - [INVALID_GRAPH_NODE_RETURN_VALUE](../troubleshooting/errors/INVALID_GRAPH_NODE_RETURN_VALUE.md)
 - [MULTIPLE_SUBGRAPHS](../troubleshooting/errors/MULTIPLE_SUBGRAPHS.md)
+- [INVALID_CHAT_HISTORY](../troubleshooting/errors/INVALID_CHAT_HISTORY.md)
 
 

docs/docs/troubleshooting/errors/INVALID_CHAT_HISTORY.md

@@ -0,0 +1,30 @@
+# INVALID_CHAT_HISTORY
+
+This error is raised in the prebuilt [create_react_agent][langgraph.prebuilt.chat_agent_executor.create_react_agent] when the `call_model` graph node receives a malformed list of messages. Specifically, it is malformed when there are `AIMessages` with `tool_calls` (LLM requesting to call a tool) that do not have a corresponding `ToolMessage` (result of a tool invocation to return to the LLM).
+
+There could be a few reasons you're seeing this error:
+
+1. You manually passed a malformed list of messages when invoking the graph, e.g. `graph.invoke({'messages': [AIMessage(..., tool_calls=[...])]})`
+2. The graph was interrupted before receiving updates from the `tools` node (i.e. a list of ToolMessages)
+and you invoked it with a an input that is not None or a ToolMessage,
+e.g. `graph.invoke({'messages': [HumanMessage(...)]}, config)`.
+    This interrupt could have been triggered in one of the following ways:
+     - You manually set `interrupt_before = ['tools']` in `create_react_agent`
+     - One of the tools raised an error that wasn't handled by the [ToolNode][langgraph.prebuilt.tool_node.ToolNode] (`"tools"`)
+
+## Troubleshooting
+
+To resolve this, you can do one of the following:
+
+1. Don't invoke the graph with a malformed list of messages
+2. In case of an interrupt (manual or due to an error) you can:
+
+    - provide ToolMessages that match existing tool calls and call `graph.invoke({'messages': [ToolMessage(...)]})`.
+    **NOTE**: this will append the messages to the history and run the graph from the START node.
+    - manually update the state and resume the graph from the interrupt:
+
+        1. get the list of most recent messages from the graph state with `graph.get_state(config)`
+        2. modify the list of messages to either remove unanswered tool calls from AIMessages
+or add ToolMessages with tool_call_ids that match unanswered tool calls
+        3. call `graph.update_state(config, {'messages': ...})` with the modified list of messages
+        4. resume the graph, e.g. call `graph.invoke(None, config)`

docs/docs/troubleshooting/errors/index.md

@@ -7,3 +7,4 @@ Errors referenced below will have an `lc_error_code` property corresponding to o
 - [INVALID_CONCURRENT_GRAPH_UPDATE](./INVALID_CONCURRENT_GRAPH_UPDATE.md)
 - [INVALID_GRAPH_NODE_RETURN_VALUE](./INVALID_GRAPH_NODE_RETURN_VALUE.md)
 - [MULTIPLE_SUBGRAPHS](./MULTIPLE_SUBGRAPHS.md)
+- [INVALID_CHAT_HISTORY](./INVALID_CHAT_HISTORY.md)

libs/langgraph/langgraph/errors.py

@@ -12,6 +12,7 @@ class ErrorCode(Enum):
     INVALID_CONCURRENT_GRAPH_UPDATE = "INVALID_CONCURRENT_GRAPH_UPDATE"
     INVALID_GRAPH_NODE_RETURN_VALUE = "INVALID_GRAPH_NODE_RETURN_VALUE"
     MULTIPLE_SUBGRAPHS = "MULTIPLE_SUBGRAPHS"
+    INVALID_CHAT_HISTORY = "INVALID_CHAT_HISTORY"
 
 
 def create_error_message(*, message: str, error_code: ErrorCode) -> str:

libs/langgraph/langgraph/prebuilt/chat_agent_executor.py

@@ -11,6 +11,7 @@ from langchain_core.tools import BaseTool
 from typing_extensions import Annotated, TypedDict
 
 from langgraph._api.deprecation import deprecated_parameter
+from langgraph.errors import ErrorCode, create_error_message
 from langgraph.graph import StateGraph
 from langgraph.graph.graph import CompiledGraph
 from langgraph.graph.message import add_messages
@@ -161,6 +162,37 @@ def _should_bind_tools(model: LanguageModelLike, tools: Sequence[BaseTool]) -> b
     return False
 
 
+def _validate_chat_history(
+    messages: Sequence[BaseMessage],
+) -> None:
+    """Validate that all tool calls in AIMessages have a corresponding ToolMessage."""
+    all_tool_calls = [
+        tool_call
+        for message in messages
+        if isinstance(message, AIMessage)
+        for tool_call in message.tool_calls
+    ]
+    tool_call_ids_with_results = {
+        message.tool_call_id for message in messages if isinstance(message, ToolMessage)
+    }
+    tool_calls_without_results = [
+        tool_call
+        for tool_call in all_tool_calls
+        if tool_call["id"] not in tool_call_ids_with_results
+    ]
+    if not tool_calls_without_results:
+        return
+
+    error_message = create_error_message(
+        message="Found AIMessages with tool_calls that do not have a corresponding ToolMessage. "
+        f"Here are the first few of those tool calls: {tool_calls_without_results[:3]}.\n\n"
+        "Every tool call (LLM requesting to call a tool) in the message history MUST have a corresponding ToolMessage "
+        "(result of a tool invocation to return to the LLM) - this is required by most LLM providers.",
+        error_code=ErrorCode.INVALID_CHAT_HISTORY,
+    )
+    raise ValueError(error_message)
+
+
 @deprecated_parameter("messages_modifier", "0.1.9", "state_modifier", removal="0.3.0")
 def create_react_agent(
     model: LanguageModelLike,
@@ -530,6 +562,7 @@ def create_react_agent(
 
     # Define the function that calls the model
     def call_model(state: AgentState, config: RunnableConfig) -> AgentState:
+        _validate_chat_history(state["messages"])
         response = model_runnable.invoke(state, config)
         has_tool_calls = isinstance(response, AIMessage) and response.tool_calls
         all_tools_return_direct = (
@@ -566,6 +599,7 @@ def create_react_agent(
         return {"messages": [response]}
 
     async def acall_model(state: AgentState, config: RunnableConfig) -> AgentState:
+        _validate_chat_history(state["messages"])
         response = await model_runnable.ainvoke(state, config)
         has_tool_calls = isinstance(response, AIMessage) and response.tool_calls
         all_tools_return_direct = (

libs/langgraph/tests/test_prebuilt.py

@@ -46,6 +46,7 @@ from langgraph.prebuilt import (
     create_react_agent,
     tools_condition,
 )
+from langgraph.prebuilt.chat_agent_executor import _validate_chat_history
 from langgraph.prebuilt.tool_node import (
     TOOL_CALL_ERROR_TEMPLATE,
     InjectedState,
@@ -378,6 +379,71 @@ def test_model_with_tools(tool_style: str):
         create_react_agent(model.bind_tools([tool1]), [tool2])
 
 
+def test__validate_messages():
+    # empty input
+    _validate_chat_history([])
+
+    # single human message
+    _validate_chat_history(
+        [
+            HumanMessage(content="What's the weather?"),
+        ]
+    )
+
+    # human + AI
+    _validate_chat_history(
+        [
+            HumanMessage(content="What's the weather?"),
+            AIMessage(content="The weather is sunny and 75°F."),
+        ]
+    )
+
+    # Answered tool calls
+    _validate_chat_history(
+        [
+            HumanMessage(content="What's the weather?"),
+            AIMessage(
+                content="Let me check that for you.",
+                tool_calls=[{"id": "call1", "name": "get_weather", "args": {}}],
+            ),
+            ToolMessage(content="Sunny, 75°F", tool_call_id="call1"),
+            AIMessage(content="The weather is sunny and 75°F."),
+        ]
+    )
+
+    # Unanswered tool calls
+    with pytest.raises(ValueError):
+        _validate_chat_history(
+            [
+                AIMessage(
+                    content="I'll check that for you.",
+                    tool_calls=[
+                        {"id": "call1", "name": "get_weather", "args": {}},
+                        {"id": "call2", "name": "get_time", "args": {}},
+                    ],
+                )
+            ]
+        )
+
+    with pytest.raises(ValueError):
+        _validate_chat_history(
+            [
+                HumanMessage(content="What's the weather and time?"),
+                AIMessage(
+                    content="I'll check that for you.",
+                    tool_calls=[
+                        {"id": "call1", "name": "get_weather", "args": {}},
+                        {"id": "call2", "name": "get_time", "args": {}},
+                    ],
+                ),
+                ToolMessage(content="Sunny, 75°F", tool_call_id="call1"),
+                AIMessage(
+                    content="The weather is sunny and 75°F. Let me check the time."
+                ),
+            ]
+        )
+
+
 def test__infer_handled_types() -> None:
     def handle(e):  # type: ignore
         return ""