From eadcec7304de1c2e9af29fc947c3455573fdfe5a Mon Sep 17 00:00:00 2001
From: Isaac Francisco <78627776+isahers1@users.noreply.github.com>
Date: Fri, 20 Sep 2024 15:12:35 -0700
Subject: [PATCH] fixed some (#1784)
---
.../langgraph_to_langgraph_cloud.ipynb | 99 +++++--------------
.../streaming-events-from-within-tools.ipynb | 95 ++++++------------
.../how-tos/streaming-from-final-node.ipynb | 30 +++---
.../streaming-tokens-without-langchain.ipynb | 73 +++++++-------
docs/docs/how-tos/streaming-tokens.ipynb | 89 +++++++++--------
5 files changed, 154 insertions(+), 232 deletions(-)
diff --git a/docs/docs/cloud/how-tos/langgraph_to_langgraph_cloud.ipynb b/docs/docs/cloud/how-tos/langgraph_to_langgraph_cloud.ipynb
index ec12d07ec..3620dc071 100644
--- a/docs/docs/cloud/how-tos/langgraph_to_langgraph_cloud.ipynb
+++ b/docs/docs/cloud/how-tos/langgraph_to_langgraph_cloud.ipynb
@@ -869,7 +869,7 @@
"id": "cb4072c9-775e-4bf5-8a1a-fb822e6de9d7",
"metadata": {},
"source": [
- "For streaming events, in LangGraph you need to use `.astream_events` method on the `CompiledGraph`. In LangGraph Cloud this is done via passing `stream_mode=\"events\"`"
+ "For streaming events, in LangGraph you need to use `.astream` method on the `CompiledGraph`. In LangGraph Cloud this is done via passing `stream_mode=\"events\"`"
]
},
{
@@ -882,89 +882,42 @@
},
{
"cell_type": "code",
- "execution_count": 2,
- "id": "94b815e4-1dd2-4999-9e73-6e29836d9160",
+ "execution_count": 22,
+ "id": "fc692060",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "Tool Calls:\n",
- " get_weather (call_UPFCSk4cQTFAuET2WgAzq0el)\n",
- " Call ID: call_UPFCSk4cQTFAuET2WgAzq0el\n",
- " Args:\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "Tool Calls:\n",
- " (None)\n",
- " Call ID: None\n",
- " Args:\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "Invalid Tool Calls:\n",
- " None (None)\n",
- " Call ID: None\n",
- " Args:\n",
- " city\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "Invalid Tool Calls:\n",
- " None (None)\n",
- " Call ID: None\n",
- " Args:\n",
- " \":\"\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "Invalid Tool Calls:\n",
- " None (None)\n",
- " Call ID: None\n",
- " Args:\n",
- " sf\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "Invalid Tool Calls:\n",
- " None (None)\n",
- " Call ID: None\n",
- " Args:\n",
- " \"}\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- "The\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- " weather\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- " in\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- " San\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- " Francisco\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- " is\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- " sunny\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- "!\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- " ☀\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n",
- "\n",
- "️\n",
- "============================\u001b[1m Aimessagechunk Message \u001b[0m============================\n"
+ "[{'name': 'get_weather', 'args': {}, 'id': 'call_G6boP6Hj21glqPTqtFdTllUd', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_weather', 'args': {}, 'id': 'call_G6boP6Hj21glqPTqtFdTllUd', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_weather', 'args': {}, 'id': 'call_G6boP6Hj21glqPTqtFdTllUd', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_weather', 'args': {'city': ''}, 'id': 'call_G6boP6Hj21glqPTqtFdTllUd', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_weather', 'args': {'city': 'sf'}, 'id': 'call_G6boP6Hj21glqPTqtFdTllUd', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_weather', 'args': {'city': 'sf'}, 'id': 'call_G6boP6Hj21glqPTqtFdTllUd', 'type': 'tool_call'}]\n",
+ "It's always sunny in sf|The| weather| in| San| Francisco| is| currently| sunny|.|"
]
}
],
"source": [
+ "from langchain_core.messages import AIMessageChunk\n",
+ "\n",
"inputs = {\"messages\": [(\"human\", \"what's the weather in sf\")]}\n",
- "async for chunk in graph.astream_events(inputs, version=\"v2\"):\n",
- " if chunk[\"event\"] == \"on_chat_model_stream\":\n",
- " chunk[\"data\"][\"chunk\"].pretty_print()"
+ "first = True\n",
+ "async for msg, metadata in graph.astream(inputs, stream_mode=\"messages\"):\n",
+ " if msg.content:\n",
+ " print(msg.content, end=\"|\", flush=True)\n",
+ "\n",
+ " if isinstance(msg, AIMessageChunk):\n",
+ " if first:\n",
+ " gathered = msg\n",
+ " first = False\n",
+ " else:\n",
+ " gathered = gathered + msg\n",
+ "\n",
+ " if msg.tool_call_chunks:\n",
+ " print(gathered.tool_calls)"
]
},
{
diff --git a/docs/docs/how-tos/streaming-events-from-within-tools.ipynb b/docs/docs/how-tos/streaming-events-from-within-tools.ipynb
index b175e5c28..27cdc27a9 100644
--- a/docs/docs/how-tos/streaming-events-from-within-tools.ipynb
+++ b/docs/docs/how-tos/streaming-events-from-within-tools.ipynb
@@ -117,7 +117,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 14,
"id": "2cb38dd9-74d8-456d-9e39-4655f2bf3f37",
"metadata": {},
"outputs": [],
@@ -157,7 +157,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 15,
"id": "7254310e-7016-45f7-9795-6d52a1160086",
"metadata": {},
"outputs": [],
@@ -177,88 +177,53 @@
},
{
"cell_type": "code",
- "execution_count": 4,
- "id": "31fe94ab-80de-4729-843e-5a0fe1bb52c0",
+ "execution_count": 25,
+ "id": "ec461f66",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "1. Books: A shelf is typically used to store books. These can be novels, textbooks, or reference books, and they come in various sizes and genres.\n",
- "\n",
- "2. Picture frames: Picture frames are commonly placed on shelves to display photographs or artwork. They can be made of wood, metal, or plastic and come in different shapes and sizes.\n",
- "\n",
- "3. Decorative items: Shelves often display decorative items such as vases, figurines, or candles. These items can add a personal touch to a room and enhance its aesthetic appeal."
+ "1|.| Books| -| A| collection| of| written| or| printed| works| bound| together| and| typically| held| upright| on| a| shelf| for| easy| access| and| storage|.\n",
+ "|2|.| Picture| frames| -| Decor|ative| frames| used| to| display| photographs| or| artwork| on| a| shelf|,| adding| a| personal| touch| to| the| space|.\n",
+ "|3|.| Decor|ative| figur|ines| -| Small| sculptures| or| statues| that| are| placed| on| a| shelf| for| decorative| purposes|,| adding| visual| interest| and| personality| to| the| room|.|"
]
}
],
"source": [
- "async for event in agent.astream_events(\n",
- " {\"messages\": [(\"human\", \"what items are on the shelf?\")]}, version=\"v2\"\n",
- "):\n",
- " tags = event.get(\"tags\", [])\n",
- " if event[\"event\"] == \"on_chat_model_stream\" and \"tool_llm\" in tags:\n",
- " print(event[\"data\"][\"chunk\"].content, end=\"\", flush=True)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "ebd8902e-935b-4724-8b5d-551b7674fd34",
- "metadata": {},
- "source": [
- "Let's inspect the last event to get the final list of messages from the agent"
+ "from langchain_core.messages import HumanMessage\n",
+ "\n",
+ "inputs = [HumanMessage(content=\"what is the weather in sf\")]\n",
+ "final_message = \"\"\n",
+ "async for msg, metadata in agent.astream({\"messages\": [(\"human\", \"what items are on the shelf?\")]}, stream_mode=\"messages\"):\n",
+ " # Stream all messages from the tool node\n",
+ " if msg.content and not isinstance(msg,HumanMessage) and metadata['langgraph_node'] == 'tools' and not msg.name:\n",
+ " print(msg.content, end=\"|\", flush=True)\n",
+ " # Final message should come from our agent\n",
+ " if msg.content and metadata['langgraph_node'] == \"agent\":\n",
+ " final_message += msg.content"
]
},
{
"cell_type": "code",
- "execution_count": 7,
- "id": "ca382c1f-b1c7-4c8a-bd9b-7a873b891b3e",
- "metadata": {},
- "outputs": [],
- "source": [
- "final_messages = event[\"data\"][\"output\"][\"messages\"]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "id": "3fa7d768-5a84-475a-950e-fd351a44841b",
+ "execution_count": 26,
+ "id": "1b35d72f",
"metadata": {},
"outputs": [
{
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "================================\u001b[1m Human Message \u001b[0m=================================\n",
- "\n",
- "what items are on the shelf?\n",
- "==================================\u001b[1m Ai Message \u001b[0m==================================\n",
- "Tool Calls:\n",
- " get_items (call_5CAMZ3asoLsZm9ocMbCOWxYQ)\n",
- " Call ID: call_5CAMZ3asoLsZm9ocMbCOWxYQ\n",
- " Args:\n",
- " place: shelf\n",
- "=================================\u001b[1m Tool Message \u001b[0m=================================\n",
- "Name: get_items\n",
- "\n",
- "1. Books - A collection of written or printed works bound together with covers. They can be fiction or non-fiction and come in various genres.\n",
- "\n",
- "2. Picture frames - A decorative border for a photograph or artwork, typically made of wood, metal, or plastic. Picture frames are used to display and protect a picture or painting.\n",
- "\n",
- "3. Candles - A cylinder of wax with a central wick that is lit to produce light or fragrance. Candles are often used for decoration, ambiance, or religious ceremonies.\n",
- "==================================\u001b[1m Ai Message \u001b[0m==================================\n",
- "\n",
- "The items on the shelf are:\n",
- "1. Books\n",
- "2. Picture frames\n",
- "3. Candles\n"
- ]
+ "data": {
+ "text/plain": [
+ "'The items on the shelf are:\\n1. Books\\n2. Picture frames\\n3. Decorative figurines'"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
}
],
"source": [
- "for message in final_messages:\n",
- " message.pretty_print()"
+ "final_message"
]
},
{
@@ -266,7 +231,7 @@
"id": "d7f9457c-5665-4cd5-9a99-d54c84270616",
"metadata": {},
"source": [
- "You can see that the content of the `ToolMessage` is the same as the output we streamed above"
+ "You can see that the content of the final message is the same as the output we streamed above"
]
}
],
diff --git a/docs/docs/how-tos/streaming-from-final-node.ipynb b/docs/docs/how-tos/streaming-from-final-node.ipynb
index 8d3b881d7..58c9a4a32 100644
--- a/docs/docs/how-tos/streaming-from-final-node.ipynb
+++ b/docs/docs/how-tos/streaming-from-final-node.ipynb
@@ -73,7 +73,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"id": "5e62618d-0e0c-483c-acd3-40a26e61894a",
"metadata": {},
"outputs": [],
@@ -120,7 +120,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 3,
"id": "8c7339d2-1835-4b5a-a99c-a60e150280af",
"metadata": {},
"outputs": [],
@@ -183,7 +183,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
"id": "2ab6d079-ba06-48ba-abe5-e72df24407af",
"metadata": {},
"outputs": [
@@ -214,7 +214,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 5,
"id": "84d65cbe-4cfe-44f8-b49e-b37632887c91",
"metadata": {},
"outputs": [],
@@ -243,30 +243,24 @@
{
"cell_type": "code",
"execution_count": 7,
- "id": "a37c3a5f-5a43-46db-940e-c583df776520",
+ "id": "68ac2c7f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Well| folks|,| looks| like| we|'ve| got| some| cloudy| skies| in| the| Big| Apple| today|.| So| grab| your| umbrella| just| in| case|,| and| don|'t| let| those| clouds| rain| on| your| parade|!|"
+ "Well| folks|,| let| me| tell| you|,| the| weather| in| San| Francisco| is| always| sunny|!| That|'s| right|,| you| can| expect| clear| skies| and| plenty| of| sunshine| when| you|'re| in| the| City| by| the| Bay|.| So| grab| your| sunglasses| and| get| ready| to| enjoy| some| beautiful| weather| in| San| Francisco|!|"
]
}
],
"source": [
- "inputs = {\"messages\": [(\"human\", \"what's the weather in nyc?\")]}\n",
- "async for event in app.astream_events(inputs, version=\"v2\"):\n",
- " kind = event[\"event\"]\n",
- " tags = event.get(\"tags\", [])\n",
- " # filter on the langgraph node name\n",
- " if kind == \"on_chat_model_stream\" and event[\"metadata\"].get(\"langgraph_node\") == \"final\":\n",
- " data = event[\"data\"]\n",
- " if data[\"chunk\"].content:\n",
- " # Empty content in the context of OpenAI or Anthropic usually means\n",
- " # that the model is asking for a tool to be invoked.\n",
- " # So we only print non-empty content\n",
- " print(data[\"chunk\"].content, end=\"|\", flush=True)"
+ "from langchain_core.messages import HumanMessage\n",
+ "\n",
+ "inputs = [HumanMessage(content=\"what is the weather in sf\")]\n",
+ "async for msg, metadata in app.astream({\"messages\": inputs}, stream_mode=\"messages\"):\n",
+ " if msg.content and not isinstance(msg,HumanMessage) and metadata['langgraph_node'] == 'final':\n",
+ " print(msg.content, end=\"|\", flush=True)"
]
},
{
diff --git a/docs/docs/how-tos/streaming-tokens-without-langchain.ipynb b/docs/docs/how-tos/streaming-tokens-without-langchain.ipynb
index 5e737cb49..797868fdf 100644
--- a/docs/docs/how-tos/streaming-tokens-without-langchain.ipynb
+++ b/docs/docs/how-tos/streaming-tokens-without-langchain.ipynb
@@ -87,7 +87,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 7,
"id": "d59234f9-173e-469d-a725-c13e0979663e",
"metadata": {},
"outputs": [],
@@ -139,7 +139,13 @@
"\n",
" if delta.content:\n",
" response_content += delta.content\n",
- " llm_run_manager.on_llm_new_token(delta.content)\n",
+ " # note: we're wrapping the response in ChatGenerationChunk so that we can stream this back using stream_mode=\"messages\"\n",
+ " chunk = ChatGenerationChunk(\n",
+ " message=AIMessageChunk(\n",
+ " content=delta.content,\n",
+ " )\n",
+ " )\n",
+ " llm_run_manager.on_llm_new_token(delta.content, chunk=chunk)\n",
"\n",
" if delta.tool_calls:\n",
" # note: for simplicity we're only handling a single tool call here\n",
@@ -147,7 +153,7 @@
" tool_call_function_name = delta.tool_calls[0].function.name\n",
" tool_call_id = delta.tool_calls[0].id\n",
"\n",
- " # note: we're wrapping the tools calls in ChatGenerationChunk so that the events from .astream_events in the graph can render tool calls correctly\n",
+ " # note: we're wrapping the tools calls in ChatGenerationChunk so that we can stream this back using stream_mode=\"messages\"\n",
" tool_call_chunk = ChatGenerationChunk(\n",
" message=AIMessageChunk(\n",
" content=\"\",\n",
@@ -189,7 +195,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 8,
"id": "b756ea32",
"metadata": {},
"outputs": [],
@@ -239,7 +245,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 9,
"id": "228260be-1f9a-4195-80e0-9604f8a5dba6",
"metadata": {},
"outputs": [],
@@ -281,47 +287,42 @@
},
{
"cell_type": "code",
- "execution_count": 4,
- "id": "45c96a79-4147-42e3-89fd-d942b2b49f6c",
+ "execution_count": 10,
+ "id": "d6ed3df5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "LLM token {'content': '', 'additional_kwargs': {'tool_calls': [{'index': 0, 'id': 'call_6XZimd7lxnCgoLK1ZM3iAR6S', 'function': {'arguments': '', 'name': 'get_items'}, 'type': 'function'}]}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [{'name': 'get_items', 'args': {}, 'id': 'call_6XZimd7lxnCgoLK1ZM3iAR6S', 'type': 'tool_call'}], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': [{'name': 'get_items', 'args': '', 'id': 'call_6XZimd7lxnCgoLK1ZM3iAR6S', 'index': 0, 'type': 'tool_call_chunk'}]}\n",
- "LLM token {'content': '', 'additional_kwargs': {'tool_calls': [{'index': 0, 'id': None, 'function': {'arguments': '{\"', 'name': None}, 'type': None}]}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [{'name': '', 'args': {}, 'id': None, 'type': 'tool_call'}], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': [{'name': None, 'args': '{\"', 'id': None, 'index': 0, 'type': 'tool_call_chunk'}]}\n",
- "LLM token {'content': '', 'additional_kwargs': {'tool_calls': [{'index': 0, 'id': None, 'function': {'arguments': 'place', 'name': None}, 'type': None}]}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [{'name': None, 'args': 'place', 'id': None, 'error': None, 'type': 'invalid_tool_call'}], 'usage_metadata': None, 'tool_call_chunks': [{'name': None, 'args': 'place', 'id': None, 'index': 0, 'type': 'tool_call_chunk'}]}\n",
- "LLM token {'content': '', 'additional_kwargs': {'tool_calls': [{'index': 0, 'id': None, 'function': {'arguments': '\":\"', 'name': None}, 'type': None}]}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [{'name': None, 'args': '\":\"', 'id': None, 'error': None, 'type': 'invalid_tool_call'}], 'usage_metadata': None, 'tool_call_chunks': [{'name': None, 'args': '\":\"', 'id': None, 'index': 0, 'type': 'tool_call_chunk'}]}\n",
- "LLM token {'content': '', 'additional_kwargs': {'tool_calls': [{'index': 0, 'id': None, 'function': {'arguments': 'bed', 'name': None}, 'type': None}]}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [{'name': None, 'args': 'bed', 'id': None, 'error': None, 'type': 'invalid_tool_call'}], 'usage_metadata': None, 'tool_call_chunks': [{'name': None, 'args': 'bed', 'id': None, 'index': 0, 'type': 'tool_call_chunk'}]}\n",
- "LLM token {'content': '', 'additional_kwargs': {'tool_calls': [{'index': 0, 'id': None, 'function': {'arguments': 'room', 'name': None}, 'type': None}]}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [{'name': None, 'args': 'room', 'id': None, 'error': None, 'type': 'invalid_tool_call'}], 'usage_metadata': None, 'tool_call_chunks': [{'name': None, 'args': 'room', 'id': None, 'index': 0, 'type': 'tool_call_chunk'}]}\n",
- "LLM token {'content': '', 'additional_kwargs': {'tool_calls': [{'index': 0, 'id': None, 'function': {'arguments': '\"}', 'name': None}, 'type': None}]}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [{'name': None, 'args': '\"}', 'id': None, 'error': None, 'type': 'invalid_tool_call'}], 'usage_metadata': None, 'tool_call_chunks': [{'name': None, 'args': '\"}', 'id': None, 'index': 0, 'type': 'tool_call_chunk'}]}\n",
- "LLM token {'content': 'In', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' the', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' bedroom', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ',', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' you', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' have', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' socks', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ',', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' shoes', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ',', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' and', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' dust', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': ' b', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': 'unn', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': 'ies', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n",
- "LLM token {'content': '.', 'additional_kwargs': {}, 'response_metadata': {}, 'type': 'AIMessageChunk', 'name': None, 'id': None, 'example': False, 'tool_calls': [], 'invalid_tool_calls': [], 'usage_metadata': None, 'tool_call_chunks': []}\n"
+ "[{'name': 'get_items', 'args': {}, 'id': 'call_h7g3jsgeRXIOUiaEC0VtM4EI', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_items', 'args': {}, 'id': 'call_h7g3jsgeRXIOUiaEC0VtM4EI', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_items', 'args': {}, 'id': 'call_h7g3jsgeRXIOUiaEC0VtM4EI', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_items', 'args': {'place': ''}, 'id': 'call_h7g3jsgeRXIOUiaEC0VtM4EI', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_items', 'args': {'place': 'bed'}, 'id': 'call_h7g3jsgeRXIOUiaEC0VtM4EI', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_items', 'args': {'place': 'bedroom'}, 'id': 'call_h7g3jsgeRXIOUiaEC0VtM4EI', 'type': 'tool_call'}]\n",
+ "[{'name': 'get_items', 'args': {'place': 'bedroom'}, 'id': 'call_h7g3jsgeRXIOUiaEC0VtM4EI', 'type': 'tool_call'}]\n",
+ "In| the| bedroom|,| you| have| socks|,| shoes|,| and| some| dust| b|unn|ies|.|"
]
}
],
"source": [
- "async for event in graph.astream_events(\n",
- " {\"messages\": [{\"role\": \"user\", \"content\": \"what's in the bedroom\"}]}, version=\"v2\"\n",
- "):\n",
- " tags = event.get(\"tags\", [])\n",
- " if event[\"event\"] == \"on_chat_model_stream\" and \"agent_llm\" in tags:\n",
- " print(\"LLM token\", event[\"data\"][\"chunk\"].dict())"
+ "from langchain_core.messages import AIMessageChunk\n",
+ "\n",
+ "first = True\n",
+ "async for msg, metadata in graph.astream({\"messages\": [{\"role\": \"user\", \"content\": \"what's in the bedroom\"}]}, stream_mode=\"messages\"):\n",
+ " if msg.content:\n",
+ " print(msg.content, end=\"|\", flush=True)\n",
+ "\n",
+ " if isinstance(msg, AIMessageChunk):\n",
+ " if first:\n",
+ " gathered = msg\n",
+ " first = False\n",
+ " else:\n",
+ " gathered = gathered + msg\n",
+ "\n",
+ " if msg.tool_call_chunks:\n",
+ " print(gathered.tool_calls)"
]
}
],
diff --git a/docs/docs/how-tos/streaming-tokens.ipynb b/docs/docs/how-tos/streaming-tokens.ipynb
index 8daf58717..b3f57d562 100644
--- a/docs/docs/how-tos/streaming-tokens.ipynb
+++ b/docs/docs/how-tos/streaming-tokens.ipynb
@@ -7,7 +7,7 @@
"source": [
"# How to stream LLM tokens from your graph\n",
"\n",
- "In this example we will stream tokens from the language model powering an agent. We will use a ReAct agent as an example. The main thing to bear in mind here is that using [async nodes](../async) typically offers the best behavior for this, since we will be using the `astream_events` method.\n",
+ "In this example we will stream tokens from the language model powering an agent. We will use a ReAct agent as an example.\n",
"\n",
"This how-to guide closely follows the others in this directory, so we will call out differences with the **STREAMING** tag below (if you just want to search for those).\n",
"\n",
@@ -22,7 +22,7 @@
" Note on Python < 3.11
\n",
" \n",
" When using python 3.8, 3.9, or 3.10, please ensure you manually pass the RunnableConfig through to the llm when invoking it like so: llm.ainvoke(..., config).\n",
- " The astream_events method collects all events from your nested code using a streaming tracer passed as a callback. In 3.11 and above, this is automatically handled via contextvar's; prior to 3.11, asyncio's tasks lacked proper contextvar support, meaning that the callbacks will only propagate if you manually pass the config through. We do this in the call_model method below.\n",
+ " The stream method collects all events from your nested code using a streaming tracer passed as a callback. In 3.11 and above, this is automatically handled via contextvar's; prior to 3.11, asyncio's tasks lacked proper contextvar support, meaning that the callbacks will only propagate if you manually pass the config through. We do this in the call_model method below.\n",
"
\n",
" "
]
@@ -108,10 +108,19 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 1,
"id": "17ef4967",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/isaachershenson/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
+ " warnings.warn(\n"
+ ]
+ }
+ ],
"source": [
"from typing import Annotated\n",
"\n",
@@ -143,7 +152,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 2,
"id": "9a8bc61e",
"metadata": {},
"outputs": [],
@@ -172,7 +181,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 3,
"id": "4d6ac180",
"metadata": {},
"outputs": [],
@@ -200,7 +209,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 4,
"id": "42c0af37",
"metadata": {},
"outputs": [],
@@ -222,7 +231,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 5,
"id": "2bbdd3bc",
"metadata": {},
"outputs": [],
@@ -263,13 +272,13 @@
"\n",
"
Manual Callback Propagation
\n",
"
\n",
- " Note that in call_model(state: State, config: RunnableConfig): below, we a) accept the RunnableConfig in the node and b) pass this in as the second arg for llm.ainvoke(..., config). This is optional for python 3.11 and later. If you ever have a problem where the LLM tokens are not streamed when using `astream_events` and you are using an older version of python, it's worth checking to ensure that the callbacks are manually propagated.
\n",
+ " Note that in
call_model(state: State, config: RunnableConfig): below, we a) accept the
RunnableConfig in the node and b) pass this in as the second arg for
llm.ainvoke(..., config). This is optional for python 3.11 and later.\n",
"