diff --git a/examples/cloud_examples/background_run.ipynb b/examples/cloud_examples/background_run.ipynb index 7199a4a17..6279402fc 100644 --- a/examples/cloud_examples/background_run.ipynb +++ b/examples/cloud_examples/background_run.ipynb @@ -108,7 +108,7 @@ ], "source": [ "# If we list runs on this thread, we can see it is empty\n", - "runs = await client.runs.list(thread['thread_id'])\n", + "runs = await client.runs.list(thread[\"thread_id\"])\n", "runs" ] }, @@ -121,7 +121,7 @@ "source": [ "# Let's kick off a run\n", "input = {\"messages\": [{\"role\": \"human\", \"content\": \"what's the weather in sf\"}]}\n", - "run = await client.runs.create(thread['thread_id'], assistant_id, input=input)" + "run = await client.runs.create(thread[\"thread_id\"], assistant_id, input=input)" ] }, { @@ -165,7 +165,7 @@ ], "source": [ "# The first time we poll it, we can see `status=pending`\n", - "await client.runs.get(thread['thread_id'], run['run_id'])" + "await client.runs.get(thread[\"thread_id\"], run[\"run_id\"])" ] }, { @@ -176,7 +176,7 @@ "outputs": [], "source": [ "# Wait until the run finishes\n", - "await client.runs.join(thread['thread_id'], run['run_id'])" + "await client.runs.join(thread[\"thread_id\"], run[\"run_id\"])" ] }, { @@ -220,7 +220,7 @@ ], "source": [ "# Eventually, it should finish and we should see `status=success`\n", - "await client.runs.get(thread['thread_id'], run['run_id'])" + "await client.runs.get(thread[\"thread_id\"], run[\"run_id\"])" ] }, { @@ -231,7 +231,7 @@ "outputs": [], "source": [ "# We can get the final results\n", - "final_result = await client.threads.get_state(thread['thread_id'])" + "final_result = await client.threads.get_state(thread[\"thread_id\"])" ] }, { @@ -360,7 +360,7 @@ ], "source": [ "# We can get the content of the final message\n", - "final_result['values']['messages'][-1]['content']" + "final_result[\"values\"][\"messages\"][-1][\"content\"]" ] } ], diff --git a/examples/cloud_examples/configuration_cloud.ipynb b/examples/cloud_examples/configuration_cloud.ipynb index 3d427bd0d..e8f00027f 100644 --- a/examples/cloud_examples/configuration_cloud.ipynb +++ b/examples/cloud_examples/configuration_cloud.ipynb @@ -68,7 +68,7 @@ "# We can do this by getting the default assistant\n", "# There should always be a default assistant with no configuration\n", "assistants = await client.assistants.search()\n", - "assistants = [a for a in assistants if not a['config']]\n", + "assistants = [a for a in assistants if not a[\"config\"]]\n", "base_assistant = assistants[0]" ] }, @@ -93,10 +93,12 @@ ], "source": [ "# We can now call `.get_schemas` to get schemas associated with this graph\n", - "schemas = await client.assistants.get_schemas(assistant_id=base_assistant[\"assistant_id\"])\n", + "schemas = await client.assistants.get_schemas(\n", + " assistant_id=base_assistant[\"assistant_id\"]\n", + ")\n", "# There are multiple types of schemas\n", "# We can get the `config_schema` to look at the the configurable parameters\n", - "schemas['config_schema']['definitions']['Configurable']['properties']" + "schemas[\"config_schema\"][\"definitions\"][\"Configurable\"][\"properties\"]" ] }, { @@ -106,7 +108,9 @@ "metadata": {}, "outputs": [], "source": [ - "assistant = await client.assistants.create(graph_id=\"agent\", config={\"configurable\": {\"model_name\": \"openai\"}})" + "assistant = await client.assistants.create(\n", + " graph_id=\"agent\", config={\"configurable\": {\"model_name\": \"openai\"}}\n", + ")" ] }, { @@ -163,7 +167,9 @@ "source": [ "thread = await client.threads.create()\n", "input = {\"messages\": [{\"role\": \"user\", \"content\": \"who made you?\"}]}\n", - "async for event in client.runs.stream(thread['thread_id'], assistant['assistant_id'], input=input):\n", + "async for event in client.runs.stream(\n", + " thread[\"thread_id\"], assistant[\"assistant_id\"], input=input\n", + "):\n", " print(event)" ] }, diff --git a/examples/cloud_examples/human-in-the-loop_cloud.ipynb b/examples/cloud_examples/human-in-the-loop_cloud.ipynb index 40f0bc116..ec305b0fa 100644 --- a/examples/cloud_examples/human-in-the-loop_cloud.ipynb +++ b/examples/cloud_examples/human-in-the-loop_cloud.ipynb @@ -112,7 +112,7 @@ } ], "source": [ - "runs = await client.runs.list(thread['thread_id'])\n", + "runs = await client.runs.list(thread[\"thread_id\"])\n", "runs" ] }, @@ -158,7 +158,11 @@ "source": [ "input = {\"messages\": [{\"role\": \"human\", \"content\": \"what's the weather in sf\"}]}\n", "async for chunk in client.runs.stream(\n", - " thread['thread_id'], assistant_id, input=input, stream_mode=\"updates\", interrupt_before=['action']\n", + " thread[\"thread_id\"],\n", + " assistant_id,\n", + " input=input,\n", + " stream_mode=\"updates\",\n", + " interrupt_before=[\"action\"],\n", "):\n", " print(f\"Receiving new event of type: {chunk.event}...\")\n", " print(chunk.data)\n", @@ -209,7 +213,11 @@ "source": [ "input = None\n", "async for chunk in client.runs.stream(\n", - " thread['thread_id'], assistant_id, input=input, stream_mode=\"updates\", interrupt_before=['action']\n", + " thread[\"thread_id\"],\n", + " assistant_id,\n", + " input=input,\n", + " stream_mode=\"updates\",\n", + " interrupt_before=[\"action\"],\n", "):\n", " print(f\"Receiving new event of type: {chunk.event}...\")\n", " print(chunk.data)\n", @@ -259,7 +267,11 @@ "source": [ "input = {\"messages\": [{\"role\": \"human\", \"content\": \"what's the weather in la?\"}]}\n", "async for chunk in client.runs.stream(\n", - " thread['thread_id'], assistant_id, input=input, stream_mode=\"updates\", interrupt_before=['action']\n", + " thread[\"thread_id\"],\n", + " assistant_id,\n", + " input=input,\n", + " stream_mode=\"updates\",\n", + " interrupt_before=[\"action\"],\n", "):\n", " print(f\"Receiving new event of type: {chunk.event}...\")\n", " print(chunk.data)\n", @@ -281,7 +293,7 @@ "metadata": {}, "outputs": [], "source": [ - "thread_state = await client.threads.get_state(thread['thread_id'])" + "thread_state = await client.threads.get_state(thread[\"thread_id\"])" ] }, { @@ -299,7 +311,7 @@ "metadata": {}, "outputs": [], "source": [ - "last_message = thread_state['values']['messages'][-1]" + "last_message = thread_state[\"values\"][\"messages\"][-1]" ] }, { @@ -323,7 +335,7 @@ } ], "source": [ - "last_message['content']" + "last_message[\"content\"]" ] }, { @@ -341,12 +353,14 @@ "metadata": {}, "outputs": [], "source": [ - "last_message['tool_calls'] = [{\n", - " 'id': last_message['tool_calls'][0]['id'],\n", - " 'name': 'tavily_search_results_json',\n", - " # We change the query to say temperature\n", - " 'args': {'query': 'weather in Louisiana'}\n", - "}]\n", + "last_message[\"tool_calls\"] = [\n", + " {\n", + " \"id\": last_message[\"tool_calls\"][0][\"id\"],\n", + " \"name\": \"tavily_search_results_json\",\n", + " # We change the query to say temperature\n", + " \"args\": {\"query\": \"weather in Louisiana\"},\n", + " }\n", + "]\n", "# last_message['content'] = [{\n", "# 'id': last_message['content'][0]['id'],\n", "# 'name': 'tavily_search_results_json',\n", @@ -383,7 +397,9 @@ } ], "source": [ - "await client.threads.update_state(thread['thread_id'], values={\"messages\": [last_message]})" + "await client.threads.update_state(\n", + " thread[\"thread_id\"], values={\"messages\": [last_message]}\n", + ")" ] }, { @@ -414,8 +430,8 @@ } ], "source": [ - "thread_state = await client.threads.get_state(thread['thread_id'])\n", - "thread_state['values']['messages'][-1]['tool_calls']" + "thread_state = await client.threads.get_state(thread[\"thread_id\"])\n", + "thread_state[\"values\"][\"messages\"][-1][\"tool_calls\"]" ] }, { @@ -462,7 +478,11 @@ "source": [ "input = None\n", "async for chunk in client.runs.stream(\n", - " thread['thread_id'], assistant_id, input=input, stream_mode=\"updates\", interrupt_before=['action']\n", + " thread[\"thread_id\"],\n", + " assistant_id,\n", + " input=input,\n", + " stream_mode=\"updates\",\n", + " interrupt_before=[\"action\"],\n", "):\n", " print(f\"Receiving new event of type: {chunk.event}...\")\n", " print(chunk.data)\n", @@ -487,7 +507,7 @@ "metadata": {}, "outputs": [], "source": [ - "thread_history = await client.threads.get_history(thread['thread_id'], limit=100)" + "thread_history = await client.threads.get_history(thread[\"thread_id\"], limit=100)" ] }, { @@ -541,7 +561,7 @@ ], "source": [ "rewind_state = thread_history[3]\n", - "rewind_state['values']['messages'][-1]['tool_calls']" + "rewind_state[\"values\"][\"messages\"][-1][\"tool_calls\"]" ] }, { @@ -563,7 +583,7 @@ } ], "source": [ - "rewind_state['config']" + "rewind_state[\"config\"]" ] }, { @@ -610,12 +630,12 @@ "source": [ "input = None\n", "async for chunk in client.runs.stream(\n", - " thread['thread_id'], \n", - " assistant_id, \n", - " input=input, \n", - " stream_mode=\"updates\", \n", - " interrupt_before=['action'],\n", - " config=rewind_state['config']\n", + " thread[\"thread_id\"],\n", + " assistant_id,\n", + " input=input,\n", + " stream_mode=\"updates\",\n", + " interrupt_before=[\"action\"],\n", + " config=rewind_state[\"config\"],\n", "):\n", " print(f\"Receiving new event of type: {chunk.event}...\")\n", " print(chunk.data)\n", diff --git a/examples/cloud_examples/reject_concurrent.ipynb b/examples/cloud_examples/reject_concurrent.ipynb index 866385680..4f8f1d97d 100644 --- a/examples/cloud_examples/reject_concurrent.ipynb +++ b/examples/cloud_examples/reject_concurrent.ipynb @@ -65,7 +65,9 @@ " await client.runs.create(\n", " thread[\"thread_id\"],\n", " assistant_id,\n", - " input={\"messages\": [{\"role\": \"human\", \"content\": \"what's the weather in nyc?\"}]},\n", + " input={\n", + " \"messages\": [{\"role\": \"human\", \"content\": \"what's the weather in nyc?\"}]\n", + " },\n", " multitask_strategy=\"reject\",\n", " )\n", "except httpx.HTTPStatusError as e:\n", diff --git a/examples/cloud_examples/same-thread.ipynb b/examples/cloud_examples/same-thread.ipynb index 94d08a04e..077ac4417 100644 --- a/examples/cloud_examples/same-thread.ipynb +++ b/examples/cloud_examples/same-thread.ipynb @@ -23,11 +23,13 @@ "\n", "client = get_client()\n", "\n", - "openai_assistant = await client.assistants.create(graph_id=\"agent\", config={\"configurable\": {\"model_name\": \"openai\"}})\n", + "openai_assistant = await client.assistants.create(\n", + " graph_id=\"agent\", config={\"configurable\": {\"model_name\": \"openai\"}}\n", + ")\n", "\n", "# There should always be a default assistant with no configuration\n", "assistants = await client.assistants.search()\n", - "default_assistant = [a for a in assistants if not a['config']][0]" + "default_assistant = [a for a in assistants if not a[\"config\"]][0]" ] }, { @@ -117,7 +119,12 @@ "source": [ "thread = await client.threads.create()\n", "input = {\"messages\": [{\"role\": \"user\", \"content\": \"who made you?\"}]}\n", - "async for event in client.runs.stream(thread['thread_id'], openai_assistant['assistant_id'], input=input, stream_mode='updates'):\n", + "async for event in client.runs.stream(\n", + " thread[\"thread_id\"],\n", + " openai_assistant[\"assistant_id\"],\n", + " input=input,\n", + " stream_mode=\"updates\",\n", + "):\n", " print(event)" ] }, @@ -147,7 +154,12 @@ ], "source": [ "input = {\"messages\": [{\"role\": \"user\", \"content\": \"and you?\"}]}\n", - "async for event in client.runs.stream(thread['thread_id'], default_assistant['assistant_id'], input=input, stream_mode='updates'):\n", + "async for event in client.runs.stream(\n", + " thread[\"thread_id\"],\n", + " default_assistant[\"assistant_id\"],\n", + " input=input,\n", + " stream_mode=\"updates\",\n", + "):\n", " print(event)" ] }, diff --git a/examples/cloud_examples/stream_messages.ipynb b/examples/cloud_examples/stream_messages.ipynb index 2fa0776e3..cf5828064 100644 --- a/examples/cloud_examples/stream_messages.ipynb +++ b/examples/cloud_examples/stream_messages.ipynb @@ -109,7 +109,7 @@ } ], "source": [ - "runs = await client.runs.list(thread['thread_id'])\n", + "runs = await client.runs.list(thread[\"thread_id\"])\n", "runs" ] }, @@ -122,11 +122,14 @@ "source": [ "# Helper function for formatting messages\n", "\n", + "\n", "def format_tool_calls(tool_calls):\n", " if tool_calls:\n", " formatted_calls = []\n", " for call in tool_calls:\n", - " formatted_calls.append(f\"Tool Call ID: {call['id']}, Function: {call['name']}, Arguments: {call['args']}\")\n", + " formatted_calls.append(\n", + " f\"Tool Call ID: {call['id']}, Function: {call['name']}, Arguments: {call['args']}\"\n", + " )\n", " return \"\\n\".join(formatted_calls)\n", " return \"No tool calls\"" ] @@ -332,36 +335,41 @@ "input = {\"messages\": [{\"role\": \"user\", \"content\": \"what's the weather in sf\"}]}\n", "config = {\"configurable\": {\"model_name\": \"openai\"}}\n", "\n", - "async for event in client.runs.stream(thread['thread_id'], assistant_id, input=input, config=config, stream_mode='messages'):\n", - " if event.event == 'metadata':\n", + "async for event in client.runs.stream(\n", + " thread[\"thread_id\"],\n", + " assistant_id,\n", + " input=input,\n", + " config=config,\n", + " stream_mode=\"messages\",\n", + "):\n", + " if event.event == \"metadata\":\n", " print(f\"Metadata: Run ID - {event.data['run_id']}\")\n", " print(\"-\" * 50)\n", - " elif event.event == 'messages/partial':\n", + " elif event.event == \"messages/partial\":\n", " for data_item in event.data:\n", - " if 'role' in data_item and data_item['role'] == 'user':\n", + " if \"role\" in data_item and data_item[\"role\"] == \"user\":\n", " print(f\"Human: {data_item['content']}\")\n", " else:\n", - " tool_calls = data_item.get('tool_calls', [])\n", - " invalid_tool_calls = data_item.get('invalid_tool_calls', [])\n", - " content = data_item.get('content', \"\")\n", - " response_metadata = data_item.get('response_metadata', {})\n", + " tool_calls = data_item.get(\"tool_calls\", [])\n", + " invalid_tool_calls = data_item.get(\"invalid_tool_calls\", [])\n", + " content = data_item.get(\"content\", \"\")\n", + " response_metadata = data_item.get(\"response_metadata\", {})\n", "\n", " if content:\n", " print(f\"AI: {content}\")\n", - " \n", + "\n", " if tool_calls:\n", " print(\"Tool Calls:\")\n", " print(format_tool_calls(tool_calls))\n", - " \n", + "\n", " if invalid_tool_calls:\n", " print(\"Invalid Tool Calls:\")\n", " print(format_tool_calls(invalid_tool_calls))\n", "\n", " if response_metadata:\n", - " finish_reason = response_metadata.get('finish_reason', 'N/A')\n", + " finish_reason = response_metadata.get(\"finish_reason\", \"N/A\")\n", " print(f\"Response Metadata: Finish Reason - {finish_reason}\")\n", - " print(\"-\" * 50)\n", - " " + " print(\"-\" * 50)" ] } ], diff --git a/examples/cloud_examples/stream_updates.ipynb b/examples/cloud_examples/stream_updates.ipynb index a8c47b4d0..41e2325f7 100644 --- a/examples/cloud_examples/stream_updates.ipynb +++ b/examples/cloud_examples/stream_updates.ipynb @@ -90,7 +90,7 @@ } ], "source": [ - "runs = await client.runs.list(thread['thread_id'])\n", + "runs = await client.runs.list(thread[\"thread_id\"])\n", "runs" ] }, @@ -134,7 +134,12 @@ ], "source": [ "input = {\"messages\": [{\"role\": \"human\", \"content\": \"what's the weather in la\"}]}\n", - "async for chunk in client.runs.stream(thread['thread_id'], assistant_id, input=input, stream_mode=\"updates\", ):\n", + "async for chunk in client.runs.stream(\n", + " thread[\"thread_id\"],\n", + " assistant_id,\n", + " input=input,\n", + " stream_mode=\"updates\",\n", + "):\n", " print(f\"Receiving new event of type: {chunk.event}...\")\n", " print(chunk.data)\n", " print(\"\\n\\n\")" diff --git a/examples/cloud_examples/stream_values.ipynb b/examples/cloud_examples/stream_values.ipynb index feaf0a1fb..e8086c773 100644 --- a/examples/cloud_examples/stream_values.ipynb +++ b/examples/cloud_examples/stream_values.ipynb @@ -93,7 +93,7 @@ "source": [ "input = {\"messages\": [{\"role\": \"human\", \"content\": \"what's the weather in la\"}]}\n", "thread = await client.threads.create()\n", - "async for chunk in client.runs.stream(thread['thread_id'], assistant_id, input=input):\n", + "async for chunk in client.runs.stream(thread[\"thread_id\"], assistant_id, input=input):\n", " print(f\"Receiving new event of type: {chunk.event}...\")\n", " print(chunk.data)\n", " print(\"\\n\\n\")" @@ -117,7 +117,7 @@ "input = {\"messages\": [{\"role\": \"human\", \"content\": \"what's the weather in la\"}]}\n", "thread = await client.threads.create()\n", "final_answer = None\n", - "async for chunk in client.runs.stream(thread['thread_id'], assistant_id, input=input):\n", + "async for chunk in client.runs.stream(thread[\"thread_id\"], assistant_id, input=input):\n", " if chunk.event == \"values\":\n", " final_answer = chunk.data" ] diff --git a/examples/learning.ipynb b/examples/learning.ipynb index 7449df414..c499cd056 100644 --- a/examples/learning.ipynb +++ b/examples/learning.ipynb @@ -303,7 +303,9 @@ "\n", "{examples}\n", "\n", - "Assist the user as they require!\"\"\".format(examples=_examples)\n", + "Assist the user as they require!\"\"\".format(\n", + " examples=_examples\n", + " )\n", "\n", " else:\n", " system_message = \"\"\"You are a helpful assistant\"\"\"\n", diff --git a/examples/llm-compiler/LLMCompiler.ipynb b/examples/llm-compiler/LLMCompiler.ipynb index 23a75ab57..daf237bfc 100644 --- a/examples/llm-compiler/LLMCompiler.ipynb +++ b/examples/llm-compiler/LLMCompiler.ipynb @@ -469,7 +469,8 @@ " args_for_tasks[task[\"idx\"]] = task[\"args\"]\n", " if (\n", " # Depends on other tasks\n", - " deps and (any([dep not in observations for dep in deps]))\n", + " deps\n", + " and (any([dep not in observations for dep in deps]))\n", " ):\n", " futures.append(\n", " executor.submit(\n", diff --git a/examples/rag/langgraph_adaptive_rag_local.ipynb b/examples/rag/langgraph_adaptive_rag_local.ipynb index 28967c81c..44e73fb90 100644 --- a/examples/rag/langgraph_adaptive_rag_local.ipynb +++ b/examples/rag/langgraph_adaptive_rag_local.ipynb @@ -146,8 +146,7 @@ "vectorstore = Chroma.from_documents(\n", " documents=doc_splits,\n", " collection_name=\"rag-chroma\",\n", - " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\",\n", - " inference_mode='local'),\n", + " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n", ")\n", "retriever = vectorstore.as_retriever()" ] diff --git a/examples/rag/langgraph_crag_local.ipynb b/examples/rag/langgraph_crag_local.ipynb index 48b68ab28..433067931 100644 --- a/examples/rag/langgraph_crag_local.ipynb +++ b/examples/rag/langgraph_crag_local.ipynb @@ -203,8 +203,10 @@ "\n", "# Embed and index\n", "if run_local == \"Yes\":\n", - " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\",\n", - " inference_mode='local',)\n", + " embedding = NomicEmbeddings(\n", + " model=\"nomic-embed-text-v1.5\",\n", + " inference_mode=\"local\",\n", + " )\n", "else:\n", " embedding = MistralAIEmbeddings(mistral_api_key=mistral_api_key)\n", "\n", diff --git a/examples/rag/langgraph_rag_agent_llama3_local.ipynb b/examples/rag/langgraph_rag_agent_llama3_local.ipynb index 038579575..f118ce137 100644 --- a/examples/rag/langgraph_rag_agent_llama3_local.ipynb +++ b/examples/rag/langgraph_rag_agent_llama3_local.ipynb @@ -120,8 +120,7 @@ "vectorstore = Chroma.from_documents(\n", " documents=doc_splits,\n", " collection_name=\"rag-chroma\",\n", - " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\",\n", - " inference_mode='local'),\n", + " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n", ")\n", "retriever = vectorstore.as_retriever()" ] diff --git a/examples/rag/langgraph_self_rag_local.ipynb b/examples/rag/langgraph_self_rag_local.ipynb index 6da5bf792..2d0af6f93 100644 --- a/examples/rag/langgraph_self_rag_local.ipynb +++ b/examples/rag/langgraph_self_rag_local.ipynb @@ -163,8 +163,7 @@ "vectorstore = Chroma.from_documents(\n", " documents=doc_splits,\n", " collection_name=\"rag-chroma\",\n", - " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\",\n", - " inference_mode='local'),\n", + " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n", ")\n", "retriever = vectorstore.as_retriever()" ] diff --git a/examples/tutorials/sql-agent.ipynb b/examples/tutorials/sql-agent.ipynb index 9a948c5ac..8d33c5009 100644 --- a/examples/tutorials/sql-agent.ipynb +++ b/examples/tutorials/sql-agent.ipynb @@ -257,6 +257,7 @@ " [RunnableLambda(handle_tool_error)], exception_key=\"error\"\n", " )\n", "\n", + "\n", "def handle_tool_error(state) -> dict:\n", " error = state.get(\"error\")\n", " tool_calls = state[\"messages\"][-1].tool_calls\n", @@ -396,6 +397,7 @@ " return \"Error: Query failed. Please rewrite your query and try again.\"\n", " return result\n", "\n", + "\n", "print(db_query_tool.invoke(\"SELECT * FROM Artist LIMIT 10;\"))" ] }, @@ -456,8 +458,12 @@ "\n", "You will call the appropriate tool to execute the query after running this check.\"\"\"\n", "\n", - "query_check_prompt = ChatPromptTemplate.from_messages([(\"system\", query_check_system),(\"placeholder\", \"{messages}\")])\n", - "query_check = query_check_prompt | ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools([db_query_tool], tool_choice=\"required\")\n", + "query_check_prompt = ChatPromptTemplate.from_messages(\n", + " [(\"system\", query_check_system), (\"placeholder\", \"{messages}\")]\n", + ")\n", + "query_check = query_check_prompt | ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools(\n", + " [db_query_tool], tool_choice=\"required\"\n", + ")\n", "\n", "query_check.invoke({\"messages\": [(\"user\", \"SELECT * FROM Artist LIMIT 10;\")]})" ] @@ -508,9 +514,11 @@ "class State(TypedDict):\n", " messages: Annotated[list[AnyMessage], add_messages]\n", "\n", + "\n", "# Define a new graph\n", "workflow = StateGraph(State)\n", "\n", + "\n", "# Add a node for the first tool call\n", "def first_tool_call(state: State) -> dict[str, list[AIMessage]]:\n", " return {\n", @@ -520,8 +528,7 @@ " tool_calls=[\n", " {\n", " \"name\": \"sql_db_list_tables\",\n", - " \"args\": {\n", - " },\n", + " \"args\": {},\n", " \"id\": \"tool_abcd123\",\n", " }\n", " ],\n", @@ -529,31 +536,41 @@ " ]\n", " }\n", "\n", + "\n", "def model_check_query(state: State) -> dict[str, list[AIMessage]]:\n", " \"\"\"\n", " Use this tool to double-check if your query is correct before executing it.\n", " \"\"\"\n", - " return {\n", - " \"messages\": [\n", - " query_check.invoke({\"messages\": [state[\"messages\"][-1]]})\n", - " ]\n", - " }\n", + " return {\"messages\": [query_check.invoke({\"messages\": [state[\"messages\"][-1]]})]}\n", + "\n", "\n", "workflow.add_node(\"first_tool_call\", first_tool_call)\n", "\n", "# Add nodes for the first two tools\n", - "workflow.add_node(\"list_tables_tool\", create_tool_node_with_fallback([list_tables_tool]))\n", + "workflow.add_node(\n", + " \"list_tables_tool\", create_tool_node_with_fallback([list_tables_tool])\n", + ")\n", "workflow.add_node(\"get_schema_tool\", create_tool_node_with_fallback([get_schema_tool]))\n", "\n", "# Add a node for a model to choose the relevant tables based on the question and available tables\n", - "model_get_schema = ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools([get_schema_tool])\n", - "workflow.add_node(\"model_get_schema\", lambda state: {\"messages\": [model_get_schema.invoke(state[\"messages\"])],})\n", + "model_get_schema = ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools(\n", + " [get_schema_tool]\n", + ")\n", + "workflow.add_node(\n", + " \"model_get_schema\",\n", + " lambda state: {\n", + " \"messages\": [model_get_schema.invoke(state[\"messages\"])],\n", + " },\n", + ")\n", + "\n", "\n", "# Describe a tool to represent the end state\n", "class SubmitFinalAnswer(BaseModel):\n", " \"\"\"Submit the final answer to the user based on the query results.\"\"\"\n", + "\n", " final_answer: str = Field(..., description=\"The final answer to the user\")\n", "\n", + "\n", "# Add a node for a model to generate a query based on the question and schema\n", "query_gen_system = \"\"\"You are a SQL expert with a strong attention to detail.\n", "\n", @@ -577,11 +594,17 @@ "If you have enough information to answer the input question, simply invoke the appropriate tool to submit the final answer to the user.\n", "\n", "DO NOT make any DML statements (INSERT, UPDATE, DELETE, DROP etc.) to the database.\"\"\"\n", - "query_gen_prompt = ChatPromptTemplate.from_messages([(\"system\", query_gen_system),(\"placeholder\", \"{messages}\")])\n", - "query_gen = query_gen_prompt | ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools([SubmitFinalAnswer])\n", + "query_gen_prompt = ChatPromptTemplate.from_messages(\n", + " [(\"system\", query_gen_system), (\"placeholder\", \"{messages}\")]\n", + ")\n", + "query_gen = query_gen_prompt | ChatOpenAI(model=\"gpt-4o\", temperature=0).bind_tools(\n", + " [SubmitFinalAnswer]\n", + ")\n", + "\n", + "\n", "def query_gen_node(state: State):\n", " message = query_gen.invoke(state)\n", - " \n", + "\n", " # Sometimes, the LLM will hallucinate and call the wrong tool. We need to catch this and return an error message.\n", " tool_messages = []\n", " if message.tool_calls:\n", @@ -597,6 +620,7 @@ " tool_messages = []\n", " return {\"messages\": [message] + tool_messages}\n", "\n", + "\n", "workflow.add_node(\"query_gen\", query_gen_node)\n", "\n", "# Add a node for the model to check the query before executing it\n", @@ -605,6 +629,7 @@ "# Add node for executing the query\n", "workflow.add_node(\"execute_query\", create_tool_node_with_fallback([db_query_tool]))\n", "\n", + "\n", "# Define a conditional edge to decide whether to continue or end the workflow\n", "def should_continue(state: State) -> Literal[END, \"correct_query\", \"query_gen\"]:\n", " messages = state[\"messages\"]\n", @@ -617,6 +642,7 @@ " else:\n", " return \"correct_query\"\n", "\n", + "\n", "# Specify the edges between the nodes\n", "workflow.set_entry_point(\"first_tool_call\")\n", "workflow.add_edge(\"first_tool_call\", \"list_tables_tool\")\n", @@ -719,9 +745,13 @@ "source": [ "import json\n", "\n", - "messages = app.invoke({\"messages\": [(\"user\", \"Which sales agent made the most in sales in 2009?\")]}) \n", - "json_str = messages['messages'][-1].additional_kwargs['tool_calls'][0]['function']['arguments']\n", - "json.loads(json_str)['final_answer']" + "messages = app.invoke(\n", + " {\"messages\": [(\"user\", \"Which sales agent made the most in sales in 2009?\")]}\n", + ")\n", + "json_str = messages[\"messages\"][-1].additional_kwargs[\"tool_calls\"][0][\"function\"][\n", + " \"arguments\"\n", + "]\n", + "json.loads(json_str)[\"final_answer\"]" ] }, { @@ -731,7 +761,9 @@ "metadata": {}, "outputs": [], "source": [ - "for event in app.stream({\"messages\": [(\"user\", \"Which sales agent made the most in sales in 2009?\")]}):\n", + "for event in app.stream(\n", + " {\"messages\": [(\"user\", \"Which sales agent made the most in sales in 2009?\")]}\n", + "):\n", " print(event)" ] }, @@ -774,8 +806,10 @@ " \"\"\"Use this for answer evaluation\"\"\"\n", " msg = {\"messages\": (\"user\", example[\"input\"])}\n", " messages = app.invoke(msg)\n", - " json_str = messages['messages'][-1].additional_kwargs['tool_calls'][0]['function']['arguments']\n", - " response = json.loads(json_str)['final_answer']\n", + " json_str = messages[\"messages\"][-1].additional_kwargs[\"tool_calls\"][0][\"function\"][\n", + " \"arguments\"\n", + " ]\n", + " response = json.loads(json_str)[\"final_answer\"]\n", " return {\"response\": response}" ] }, @@ -792,13 +826,14 @@ "# Grade prompt\n", "grade_prompt_answer_accuracy = prompt = hub.pull(\"langchain-ai/rag-answer-vs-reference\")\n", "\n", + "\n", "def answer_evaluator(run, example) -> dict:\n", " \"\"\"\n", " A simple evaluator for RAG answer accuracy\n", " \"\"\"\n", "\n", " # Get question, ground truth answer, chain\n", - " input_question = example.inputs[\"input\"] \n", + " input_question = example.inputs[\"input\"]\n", " reference = example.outputs[\"output\"]\n", " prediction = run.outputs[\"response\"]\n", "\n", @@ -809,9 +844,13 @@ " answer_grader = grade_prompt_answer_accuracy | llm\n", "\n", " # Run evaluator\n", - " score = answer_grader.invoke({\"question\": input_question,\n", - " \"correct_answer\": reference,\n", - " \"student_answer\": prediction})\n", + " score = answer_grader.invoke(\n", + " {\n", + " \"question\": input_question,\n", + " \"correct_answer\": reference,\n", + " \"student_answer\": prediction,\n", + " }\n", + " )\n", " score = score[\"Score\"]\n", "\n", " return {\"key\": \"answer_v_reference_score\", \"score\": score}" @@ -872,10 +911,12 @@ "outputs": [], "source": [ "# These are the tools that we expect the agent to use\n", - "expected_trajectory=['sql_db_list_tables', # first: list_tables_tool node\n", - " 'sql_db_schema', # second: get_schema_tool node\n", - " 'db_query_tool', # third: execute_query node\n", - " 'SubmitFinalAnswer'] # fourth: query_gen" + "expected_trajectory = [\n", + " \"sql_db_list_tables\", # first: list_tables_tool node\n", + " \"sql_db_schema\", # second: get_schema_tool node\n", + " \"db_query_tool\", # third: execute_query node\n", + " \"SubmitFinalAnswer\",\n", + "] # fourth: query_gen" ] }, { @@ -903,48 +944,59 @@ "\n", "\n", "def find_tool_calls(messages):\n", - " \"\"\" \n", - " Find all tool calls in the messages returned \n", " \"\"\"\n", - " tool_calls = [tc['name'] for m in messages['messages'] for tc in getattr(m, 'tool_calls', [])]\n", + " Find all tool calls in the messages returned\n", + " \"\"\"\n", + " tool_calls = [\n", + " tc[\"name\"] for m in messages[\"messages\"] for tc in getattr(m, \"tool_calls\", [])\n", + " ]\n", " return tool_calls\n", "\n", - "def contains_all_tool_calls_in_order_exact_match(root_run: Run, example: Example) -> dict:\n", + "\n", + "def contains_all_tool_calls_in_order_exact_match(\n", + " root_run: Run, example: Example\n", + ") -> dict:\n", " \"\"\"\n", " Check if all expected tools are called in exact order and without any additional tool calls.\n", " \"\"\"\n", - " expected_trajectory = ['sql_db_list_tables', 'sql_db_schema', 'db_query_tool', 'SubmitFinalAnswer']\n", + " expected_trajectory = [\n", + " \"sql_db_list_tables\",\n", + " \"sql_db_schema\",\n", + " \"db_query_tool\",\n", + " \"SubmitFinalAnswer\",\n", + " ]\n", " messages = root_run.outputs[\"response\"]\n", " tool_calls = find_tool_calls(messages)\n", - " \n", + "\n", " # Print the tool calls for debugging\n", " print(\"Here are my tool calls:\")\n", " print(tool_calls)\n", - " \n", + "\n", " # Check if the tool calls match the expected trajectory exactly\n", " if tool_calls == expected_trajectory:\n", " score = 1\n", " else:\n", " score = 0\n", - " \n", + "\n", " return {\"score\": int(score), \"key\": \"multi_tool_call_in_exact_order\"}\n", "\n", + "\n", "def contains_all_tool_calls_in_order(root_run: Run, example: Example) -> dict:\n", " \"\"\"\n", - " Check if all expected tools are called in order, \n", + " Check if all expected tools are called in order,\n", " but it allows for other tools to be called in between the expected ones.\n", " \"\"\"\n", " messages = root_run.outputs[\"response\"]\n", " tool_calls = find_tool_calls(messages)\n", - " \n", + "\n", " # Print the tool calls for debugging\n", " print(\"Here are my tool calls:\")\n", " print(tool_calls)\n", - " \n", + "\n", " it = iter(tool_calls)\n", " if all(elem in it for elem in expected_trajectory):\n", " score = 1\n", - " else: \n", + " else:\n", " score = 0\n", " return {\"score\": int(score), \"key\": \"multi_tool_call_in_order\"}" ] @@ -959,7 +1011,10 @@ "experiment_results = evaluate(\n", " predict_sql_agent_messages,\n", " data=dataset_name,\n", - " evaluators=[contains_all_tool_calls_in_order,contains_all_tool_calls_in_order_exact_match],\n", + " evaluators=[\n", + " contains_all_tool_calls_in_order,\n", + " contains_all_tool_calls_in_order_exact_match,\n", + " ],\n", " num_repetitions=3,\n", " experiment_prefix=\"sql-agent-multi-step-tool-calling-trajecory-in-order\",\n", " metadata={\"version\": \"Chinook, gpt-4o multi-step-agent\"},\n", diff --git a/examples/tutorials/tnt-llm/tnt-llm.ipynb b/examples/tutorials/tnt-llm/tnt-llm.ipynb index 39026fd92..e92e99a5d 100644 --- a/examples/tutorials/tnt-llm/tnt-llm.ipynb +++ b/examples/tutorials/tnt-llm/tnt-llm.ipynb @@ -153,7 +153,9 @@ "\n", "\n", "summary_llm_chain = (\n", - " summary_prompt | ChatAnthropic(model=\"claude-3-haiku-20240307\") | StrOutputParser()\n", + " summary_prompt\n", + " | ChatAnthropic(model=\"claude-3-haiku-20240307\")\n", + " | StrOutputParser()\n", " # Customize the tracing name for easier organization\n", ").with_config(run_name=\"GenerateSummary\")\n", "summary_chain = summary_llm_chain | parse_summary\n",