From d1f06d677118f14a8b8a1a51f4e0ab4d1034e7ef Mon Sep 17 00:00:00 2001
From: Isaac Francisco <78627776+isahers1@users.noreply.github.com>
Date: Thu, 5 Sep 2024 14:46:37 -0700
Subject: [PATCH] removing empty cells (#1624)
---
examples/async.ipynb | 186 ++++-
.../langgraph_code_assistant.ipynb | 510 +++++++++++-
examples/human_in_the_loop/time-travel.ipynb | 50 +-
.../multi-agent-collaboration.ipynb | 238 +++++-
.../plan-and-execute/plan-and-execute.ipynb | 240 +++++-
examples/rag/langgraph_adaptive_rag.ipynb | 500 ++++++++++-
.../rag/langgraph_adaptive_rag_local.ipynb | 433 +++++++++-
examples/rag/langgraph_crag.ipynb | 391 ++++++++-
examples/rag/langgraph_crag_local.ipynb | 306 ++++++-
examples/rag/langgraph_self_rag.ipynb | 456 +++++++++-
examples/rag/langgraph_self_rag_local.ipynb | 410 ++++++++-
.../streaming-tokens-without-langchain.ipynb | 188 ++++-
examples/usaco/usaco.ipynb | 779 ++++++++++++++++--
13 files changed, 4383 insertions(+), 304 deletions(-)
diff --git a/examples/async.ipynb b/examples/async.ipynb
index f124292de..e3e3e27b0 100644
--- a/examples/async.ipynb
+++ b/examples/async.ipynb
@@ -37,7 +37,10 @@
"id": "af4ce0ba-7596-4e5f-8bf8-0b0bd6e62833",
"metadata": {},
"outputs": [],
- "source": ["%%capture --no-stderr\n%pip install --quiet -U langgraph langchain_anthropic"]
+ "source": [
+ "%%capture --no-stderr\n",
+ "%pip install --quiet -U langgraph langchain_anthropic"
+ ]
},
{
"cell_type": "markdown",
@@ -53,7 +56,18 @@
"id": "c903a1cf-2977-4e2d-ad7d-8b3946821d89",
"metadata": {},
"outputs": [],
- "source": ["import getpass\nimport os\n\n\ndef _set_env(var: str):\n if not os.environ.get(var):\n os.environ[var] = getpass.getpass(f\"{var}: \")\n\n\n_set_env(\"ANTHROPIC_API_KEY\")"]
+ "source": [
+ "import getpass\n",
+ "import os\n",
+ "\n",
+ "\n",
+ "def _set_env(var: str):\n",
+ " if not os.environ.get(var):\n",
+ " os.environ[var] = getpass.getpass(f\"{var}: \")\n",
+ "\n",
+ "\n",
+ "_set_env(\"ANTHROPIC_API_KEY\")"
+ ]
},
{
"cell_type": "markdown",
@@ -69,7 +83,10 @@
"id": "95e25aec-7c9f-4a63-b143-225d0e9a79c3",
"metadata": {},
"outputs": [],
- "source": ["os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n_set_env(\"LANGCHAIN_API_KEY\")"]
+ "source": [
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "_set_env(\"LANGCHAIN_API_KEY\")"
+ ]
},
{
"cell_type": "markdown",
@@ -95,7 +112,22 @@
"id": "6768a3ab",
"metadata": {},
"outputs": [],
- "source": ["from typing import Annotated\n\nfrom typing_extensions import TypedDict\n\nfrom langgraph.graph.message import add_messages\n\n# Add messages essentially does this with more\n# robust handling\n# def add_messages(left: list, right: list):\n# return left + right\n\n\nclass State(TypedDict):\n messages: Annotated[list, add_messages]"]
+ "source": [
+ "from typing import Annotated\n",
+ "\n",
+ "from typing_extensions import TypedDict\n",
+ "\n",
+ "from langgraph.graph.message import add_messages\n",
+ "\n",
+ "# Add messages essentially does this with more\n",
+ "# robust handling\n",
+ "# def add_messages(left: list, right: list):\n",
+ "# return left + right\n",
+ "\n",
+ "\n",
+ "class State(TypedDict):\n",
+ " messages: Annotated[list, add_messages]"
+ ]
},
{
"cell_type": "markdown",
@@ -115,7 +147,19 @@
"id": "d7ef57dd-5d6e-4ad3-9377-a92201c1310e",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.tools import tool\n\n\n@tool\ndef search(query: str):\n \"\"\"Call to surf the web.\"\"\"\n # This is a placeholder, but don't tell the LLM that...\n return [\"The answer to your question lies within.\"]\n\n\ntools = [search]"]
+ "source": [
+ "from langchain_core.tools import tool\n",
+ "\n",
+ "\n",
+ "@tool\n",
+ "def search(query: str):\n",
+ " \"\"\"Call to surf the web.\"\"\"\n",
+ " # This is a placeholder, but don't tell the LLM that...\n",
+ " return [\"The answer to your question lies within.\"]\n",
+ "\n",
+ "\n",
+ "tools = [search]"
+ ]
},
{
"cell_type": "markdown",
@@ -132,7 +176,11 @@
"id": "5cf3331e-ccb3-41c8-aeb9-a840a94d41e7",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.prebuilt import ToolNode\n\ntool_node = ToolNode(tools)"]
+ "source": [
+ "from langgraph.prebuilt import ToolNode\n",
+ "\n",
+ "tool_node = ToolNode(tools)"
+ ]
},
{
"cell_type": "markdown",
@@ -156,7 +204,11 @@
"id": "892b54b9-75f0-4804-9ed0-88b5e5532989",
"metadata": {},
"outputs": [],
- "source": ["from langchain_anthropic import ChatAnthropic\n\nmodel = ChatAnthropic(model=\"claude-3-haiku-20240307\")"]
+ "source": [
+ "from langchain_anthropic import ChatAnthropic\n",
+ "\n",
+ "model = ChatAnthropic(model=\"claude-3-haiku-20240307\")"
+ ]
},
{
"cell_type": "markdown",
@@ -174,7 +226,9 @@
"id": "cd3cbae5-d92c-4559-a4aa-44721b80d107",
"metadata": {},
"outputs": [],
- "source": ["model = model.bind_tools(tools)"]
+ "source": [
+ "model = model.bind_tools(tools)"
+ ]
},
{
"cell_type": "markdown",
@@ -213,7 +267,29 @@
"id": "3b541bb9-900c-40d0-964d-7b5dfee30667",
"metadata": {},
"outputs": [],
- "source": ["from typing import Literal\n\n\n# Define the function that determines whether to continue or not\ndef should_continue(state: State) -> Literal[\"end\", \"continue\"]:\n messages = state[\"messages\"]\n last_message = messages[-1]\n # If there is no tool call, then we finish\n if not last_message.tool_calls:\n return \"end\"\n # Otherwise if there is, we continue\n else:\n return \"continue\"\n\n\n# Define the function that calls the model\nasync def call_model(state: State):\n messages = state[\"messages\"]\n response = await model.ainvoke(messages)\n # We return a list, because this will get added to the existing list\n return {\"messages\": [response]}"]
+ "source": [
+ "from typing import Literal\n",
+ "\n",
+ "\n",
+ "# Define the function that determines whether to continue or not\n",
+ "def should_continue(state: State) -> Literal[\"end\", \"continue\"]:\n",
+ " messages = state[\"messages\"]\n",
+ " last_message = messages[-1]\n",
+ " # If there is no tool call, then we finish\n",
+ " if not last_message.tool_calls:\n",
+ " return \"end\"\n",
+ " # Otherwise if there is, we continue\n",
+ " else:\n",
+ " return \"continue\"\n",
+ "\n",
+ "\n",
+ "# Define the function that calls the model\n",
+ "async def call_model(state: State):\n",
+ " messages = state[\"messages\"]\n",
+ " response = await model.ainvoke(messages)\n",
+ " # We return a list, because this will get added to the existing list\n",
+ " return {\"messages\": [response]}"
+ ]
},
{
"cell_type": "markdown",
@@ -231,7 +307,50 @@
"id": "813ae66c-3b58-4283-a02a-36da72a2ab90",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.graph import END, StateGraph, START\n\n# Define a new graph\nworkflow = StateGraph(State)\n\n# Define the two nodes we will cycle between\nworkflow.add_node(\"agent\", call_model)\nworkflow.add_node(\"action\", tool_node)\n\n# Set the entrypoint as `agent`\n# This means that this node is the first one called\nworkflow.add_edge(START, \"agent\")\n\n# We now add a conditional edge\nworkflow.add_conditional_edges(\n # First, we define the start node. We use `agent`.\n # This means these are the edges taken after the `agent` node is called.\n \"agent\",\n # Next, we pass in the function that will determine which node is called next.\n should_continue,\n # Finally we pass in a mapping.\n # The keys are strings, and the values are other nodes.\n # END is a special node marking that the graph should finish.\n # What will happen is we will call `should_continue`, and then the output of that\n # will be matched against the keys in this mapping.\n # Based on which one it matches, that node will then be called.\n {\n # If `tools`, then we call the tool node.\n \"continue\": \"action\",\n # Otherwise we finish.\n \"end\": END,\n },\n)\n\n# We now add a normal edge from `tools` to `agent`.\n# This means that after `tools` is called, `agent` node is called next.\nworkflow.add_edge(\"action\", \"agent\")\n\n# Finally, we compile it!\n# This compiles it into a LangChain Runnable,\n# meaning you can use it as you would any other runnable\napp = workflow.compile()"]
+ "source": [
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "# Define a new graph\n",
+ "workflow = StateGraph(State)\n",
+ "\n",
+ "# Define the two nodes we will cycle between\n",
+ "workflow.add_node(\"agent\", call_model)\n",
+ "workflow.add_node(\"action\", tool_node)\n",
+ "\n",
+ "# Set the entrypoint as `agent`\n",
+ "# This means that this node is the first one called\n",
+ "workflow.add_edge(START, \"agent\")\n",
+ "\n",
+ "# We now add a conditional edge\n",
+ "workflow.add_conditional_edges(\n",
+ " # First, we define the start node. We use `agent`.\n",
+ " # This means these are the edges taken after the `agent` node is called.\n",
+ " \"agent\",\n",
+ " # Next, we pass in the function that will determine which node is called next.\n",
+ " should_continue,\n",
+ " # Finally we pass in a mapping.\n",
+ " # The keys are strings, and the values are other nodes.\n",
+ " # END is a special node marking that the graph should finish.\n",
+ " # What will happen is we will call `should_continue`, and then the output of that\n",
+ " # will be matched against the keys in this mapping.\n",
+ " # Based on which one it matches, that node will then be called.\n",
+ " {\n",
+ " # If `tools`, then we call the tool node.\n",
+ " \"continue\": \"action\",\n",
+ " # Otherwise we finish.\n",
+ " \"end\": END,\n",
+ " },\n",
+ ")\n",
+ "\n",
+ "# We now add a normal edge from `tools` to `agent`.\n",
+ "# This means that after `tools` is called, `agent` node is called next.\n",
+ "workflow.add_edge(\"action\", \"agent\")\n",
+ "\n",
+ "# Finally, we compile it!\n",
+ "# This compiles it into a LangChain Runnable,\n",
+ "# meaning you can use it as you would any other runnable\n",
+ "app = workflow.compile()"
+ ]
},
{
"cell_type": "code",
@@ -250,7 +369,11 @@
"output_type": "display_data"
}
],
- "source": ["from IPython.display import Image, display\n\ndisplay(Image(app.get_graph().draw_mermaid_png()))"]
+ "source": [
+ "from IPython.display import Image, display\n",
+ "\n",
+ "display(Image(app.get_graph().draw_mermaid_png()))"
+ ]
},
{
"cell_type": "markdown",
@@ -283,7 +406,12 @@
"output_type": "execute_result"
}
],
- "source": ["from langchain_core.messages import HumanMessage\n\ninputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\nawait app.ainvoke(inputs)"]
+ "source": [
+ "from langchain_core.messages import HumanMessage\n",
+ "\n",
+ "inputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\n",
+ "await app.ainvoke(inputs)"
+ ]
},
{
"cell_type": "markdown",
@@ -352,7 +480,16 @@
]
}
],
- "source": ["inputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\nasync for output in app.astream(inputs, stream_mode=\"updates\"):\n # stream_mode=\"updates\" yields dictionaries with output keyed by node name\n for key, value in output.items():\n print(f\"Output from node '{key}':\")\n print(\"---\")\n print(value[\"messages\"][-1].pretty_print())\n print(\"\\n---\\n\")"]
+ "source": [
+ "inputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\n",
+ "async for output in app.astream(inputs, stream_mode=\"updates\"):\n",
+ " # stream_mode=\"updates\" yields dictionaries with output keyed by node name\n",
+ " for key, value in output.items():\n",
+ " print(f\"Output from node '{key}':\")\n",
+ " print(\"---\")\n",
+ " print(value[\"messages\"][-1].pretty_print())\n",
+ " print(\"\\n---\\n\")"
+ ]
},
{
"cell_type": "markdown",
@@ -409,15 +546,20 @@
]
}
],
- "source": ["inputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\nasync for output in app.astream_log(inputs, include_types=[\"llm\"]):\n # astream_log() yields the requested logs (here LLMs) in JSONPatch format\n for op in output.ops:\n if op[\"path\"] == \"/streamed_output/-\":\n # this is the output from .stream()\n ...\n elif op[\"path\"].startswith(\"/logs/\") and op[\"path\"].endswith(\n \"/streamed_output/-\"\n ):\n # because we chose to only include LLMs, these are LLM tokens\n print(op[\"value\"].content, end=\"|\")"]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "08ae8246-11d5-40e1-8567-361e5bef8917",
- "metadata": {},
- "outputs": [],
- "source": [""]
+ "source": [
+ "inputs = {\"messages\": [HumanMessage(content=\"what is the weather in sf\")]}\n",
+ "async for output in app.astream_log(inputs, include_types=[\"llm\"]):\n",
+ " # astream_log() yields the requested logs (here LLMs) in JSONPatch format\n",
+ " for op in output.ops:\n",
+ " if op[\"path\"] == \"/streamed_output/-\":\n",
+ " # this is the output from .stream()\n",
+ " ...\n",
+ " elif op[\"path\"].startswith(\"/logs/\") and op[\"path\"].endswith(\n",
+ " \"/streamed_output/-\"\n",
+ " ):\n",
+ " # because we chose to only include LLMs, these are LLM tokens\n",
+ " print(op[\"value\"].content, end=\"|\")"
+ ]
}
],
"metadata": {
diff --git a/examples/code_assistant/langgraph_code_assistant.ipynb b/examples/code_assistant/langgraph_code_assistant.ipynb
index 2115a370f..1dab64f30 100644
--- a/examples/code_assistant/langgraph_code_assistant.ipynb
+++ b/examples/code_assistant/langgraph_code_assistant.ipynb
@@ -34,7 +34,9 @@
"id": "e3900420",
"metadata": {},
"outputs": [],
- "source": ["! pip install -U langchain_community langchain-openai langchain-anthropic langchain langgraph bs4"]
+ "source": [
+ "! pip install -U langchain_community langchain-openai langchain-anthropic langchain langgraph bs4"
+ ]
},
{
"cell_type": "markdown",
@@ -52,7 +54,24 @@
"id": "c2eb35d1-4990-47dc-a5c4-208bae588a82",
"metadata": {},
"outputs": [],
- "source": ["from bs4 import BeautifulSoup as Soup\nfrom langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n\n# LCEL docs\nurl = \"https://python.langchain.com/v0.2/docs/concepts/#langchain-expression-language-lcel\"\nloader = RecursiveUrlLoader(\n url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n)\ndocs = loader.load()\n\n# Sort the list based on the URLs and get the text\nd_sorted = sorted(docs, key=lambda x: x.metadata[\"source\"])\nd_reversed = list(reversed(d_sorted))\nconcatenated_content = \"\\n\\n\\n --- \\n\\n\\n\".join(\n [doc.page_content for doc in d_reversed]\n)"]
+ "source": [
+ "from bs4 import BeautifulSoup as Soup\n",
+ "from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n",
+ "\n",
+ "# LCEL docs\n",
+ "url = \"https://python.langchain.com/v0.2/docs/concepts/#langchain-expression-language-lcel\"\n",
+ "loader = RecursiveUrlLoader(\n",
+ " url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n",
+ ")\n",
+ "docs = loader.load()\n",
+ "\n",
+ "# Sort the list based on the URLs and get the text\n",
+ "d_sorted = sorted(docs, key=lambda x: x.metadata[\"source\"])\n",
+ "d_reversed = list(reversed(d_sorted))\n",
+ "concatenated_content = \"\\n\\n\\n --- \\n\\n\\n\".join(\n",
+ " [doc.page_content for doc in d_reversed]\n",
+ ")"
+ ]
},
{
"cell_type": "markdown",
@@ -74,7 +93,45 @@
"id": "3ba3df70-f6b4-4ea5-a210-e10944960bc6",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\n### OpenAI\n\n# Grader prompt\ncode_gen_prompt = ChatPromptTemplate.from_messages(\n [\n (\n \"system\",\n \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n Here is a full set of LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user \n question based on the above provided documentation. Ensure any code you provide can be executed \\n \n with all required imports and variables defined. Structure your answer with a description of the code solution. \\n\n Then list the imports. And finally list the functioning code block. Here is the user question:\"\"\",\n ),\n (\"placeholder\", \"{messages}\"),\n ]\n)\n\n\n# Data model\nclass code(BaseModel):\n \"\"\"Code output\"\"\"\n\n prefix: str = Field(description=\"Description of the problem and approach\")\n imports: str = Field(description=\"Code block import statements\")\n code: str = Field(description=\"Code block not including import statements\")\n description = \"Schema for code solutions to questions about LCEL.\"\n\n\nexpt_llm = \"gpt-4-0125-preview\"\nllm = ChatOpenAI(temperature=0, model=expt_llm)\ncode_gen_chain = code_gen_prompt | llm.with_structured_output(code)\nquestion = \"How do I build a RAG chain in LCEL?\"\n# solution = code_gen_chain_oai.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})"]
+ "source": [
+ "from langchain_core.prompts import ChatPromptTemplate\n",
+ "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "### OpenAI\n",
+ "\n",
+ "# Grader prompt\n",
+ "code_gen_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\n",
+ " \"system\",\n",
+ " \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
+ " Here is a full set of LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user \n",
+ " question based on the above provided documentation. Ensure any code you provide can be executed \\n \n",
+ " with all required imports and variables defined. Structure your answer with a description of the code solution. \\n\n",
+ " Then list the imports. And finally list the functioning code block. Here is the user question:\"\"\",\n",
+ " ),\n",
+ " (\"placeholder\", \"{messages}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class code(BaseModel):\n",
+ " \"\"\"Code output\"\"\"\n",
+ "\n",
+ " prefix: str = Field(description=\"Description of the problem and approach\")\n",
+ " imports: str = Field(description=\"Code block import statements\")\n",
+ " code: str = Field(description=\"Code block not including import statements\")\n",
+ " description = \"Schema for code solutions to questions about LCEL.\"\n",
+ "\n",
+ "\n",
+ "expt_llm = \"gpt-4-0125-preview\"\n",
+ "llm = ChatOpenAI(temperature=0, model=expt_llm)\n",
+ "code_gen_chain = code_gen_prompt | llm.with_structured_output(code)\n",
+ "question = \"How do I build a RAG chain in LCEL?\"\n",
+ "# solution = code_gen_chain_oai.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})"
+ ]
},
{
"cell_type": "code",
@@ -82,7 +139,118 @@
"id": "cd30b67d-96db-4e51-a540-ae23fcc1f878",
"metadata": {},
"outputs": [],
- "source": ["from langchain_anthropic import ChatAnthropic\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\n\n### Anthropic\n\n# Prompt to enforce tool use\ncode_gen_prompt_claude = ChatPromptTemplate.from_messages(\n [\n (\n \"system\",\n \"\"\" You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n Here is the LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user question based on the \\n \n above provided documentation. Ensure any code you provide can be executed with all required imports and variables \\n\n defined. Structure your answer: 1) a prefix describing the code solution, 2) the imports, 3) the functioning code block. \\n\n Invoke the code tool to structure the output correctly. \\n Here is the user question:\"\"\",\n ),\n (\"placeholder\", \"{messages}\"),\n ]\n)\n\n\n# Data model\nclass code(BaseModel):\n \"\"\"Code output\"\"\"\n\n prefix: str = Field(description=\"Description of the problem and approach\")\n imports: str = Field(description=\"Code block import statements\")\n code: str = Field(description=\"Code block not including import statements\")\n description = \"Schema for code solutions to questions about LCEL.\"\n\n\n# LLM\n# expt_llm = \"claude-3-haiku-20240307\"\nexpt_llm = \"claude-3-opus-20240229\"\nllm = ChatAnthropic(\n model=expt_llm,\n default_headers={\"anthropic-beta\": \"tools-2024-04-04\"},\n)\n\nstructured_llm_claude = llm.with_structured_output(code, include_raw=True)\n\n\n# Optional: Check for errors in case tool use is flaky\ndef check_claude_output(tool_output):\n \"\"\"Check for parse error or failure to call the tool\"\"\"\n\n # Error with parsing\n if tool_output[\"parsing_error\"]:\n # Report back output and parsing errors\n print(\"Parsing error!\")\n raw_output = str(tool_output[\"raw\"].content)\n error = tool_output[\"parsing_error\"]\n raise ValueError(\n f\"Error parsing your output! Be sure to invoke the tool. Output: {raw_output}. \\n Parse error: {error}\"\n )\n\n # Tool was not invoked\n elif not tool_output[\"parsed\"]:\n print(\"Failed to invoke tool!\")\n raise ValueError(\n \"You did not use the provided tool! Be sure to invoke the tool to structure the output.\"\n )\n return tool_output\n\n\n# Chain with output check\ncode_chain_claude_raw = (\n code_gen_prompt_claude | structured_llm_claude | check_claude_output\n)\n\n\ndef insert_errors(inputs):\n \"\"\"Insert errors for tool parsing in the messages\"\"\"\n\n # Get errors\n error = inputs[\"error\"]\n messages = inputs[\"messages\"]\n messages += [\n (\n \"assistant\",\n f\"Retry. You are required to fix the parsing errors: {error} \\n\\n You must invoke the provided tool.\",\n )\n ]\n return {\n \"messages\": messages,\n \"context\": inputs[\"context\"],\n }\n\n\n# This will be run as a fallback chain\nfallback_chain = insert_errors | code_chain_claude_raw\nN = 3 # Max re-tries\ncode_gen_chain_re_try = code_chain_claude_raw.with_fallbacks(\n fallbacks=[fallback_chain] * N, exception_key=\"error\"\n)\n\n\ndef parse_output(solution):\n \"\"\"When we add 'include_raw=True' to structured output,\n it will return a dict w 'raw', 'parsed', 'parsing_error'.\"\"\"\n\n return solution[\"parsed\"]\n\n\n# Optional: With re-try to correct for failure to invoke tool\ncode_gen_chain = code_gen_chain_re_try | parse_output\n\n# No re-try\ncode_gen_chain = code_gen_prompt_claude | structured_llm_claude | parse_output"]
+ "source": [
+ "from langchain_anthropic import ChatAnthropic\n",
+ "from langchain_core.prompts import ChatPromptTemplate\n",
+ "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+ "\n",
+ "### Anthropic\n",
+ "\n",
+ "# Prompt to enforce tool use\n",
+ "code_gen_prompt_claude = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\n",
+ " \"system\",\n",
+ " \"\"\" You are a coding assistant with expertise in LCEL, LangChain expression language. \\n \n",
+ " Here is the LCEL documentation: \\n ------- \\n {context} \\n ------- \\n Answer the user question based on the \\n \n",
+ " above provided documentation. Ensure any code you provide can be executed with all required imports and variables \\n\n",
+ " defined. Structure your answer: 1) a prefix describing the code solution, 2) the imports, 3) the functioning code block. \\n\n",
+ " Invoke the code tool to structure the output correctly. \\n Here is the user question:\"\"\",\n",
+ " ),\n",
+ " (\"placeholder\", \"{messages}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class code(BaseModel):\n",
+ " \"\"\"Code output\"\"\"\n",
+ "\n",
+ " prefix: str = Field(description=\"Description of the problem and approach\")\n",
+ " imports: str = Field(description=\"Code block import statements\")\n",
+ " code: str = Field(description=\"Code block not including import statements\")\n",
+ " description = \"Schema for code solutions to questions about LCEL.\"\n",
+ "\n",
+ "\n",
+ "# LLM\n",
+ "# expt_llm = \"claude-3-haiku-20240307\"\n",
+ "expt_llm = \"claude-3-opus-20240229\"\n",
+ "llm = ChatAnthropic(\n",
+ " model=expt_llm,\n",
+ " default_headers={\"anthropic-beta\": \"tools-2024-04-04\"},\n",
+ ")\n",
+ "\n",
+ "structured_llm_claude = llm.with_structured_output(code, include_raw=True)\n",
+ "\n",
+ "\n",
+ "# Optional: Check for errors in case tool use is flaky\n",
+ "def check_claude_output(tool_output):\n",
+ " \"\"\"Check for parse error or failure to call the tool\"\"\"\n",
+ "\n",
+ " # Error with parsing\n",
+ " if tool_output[\"parsing_error\"]:\n",
+ " # Report back output and parsing errors\n",
+ " print(\"Parsing error!\")\n",
+ " raw_output = str(tool_output[\"raw\"].content)\n",
+ " error = tool_output[\"parsing_error\"]\n",
+ " raise ValueError(\n",
+ " f\"Error parsing your output! Be sure to invoke the tool. Output: {raw_output}. \\n Parse error: {error}\"\n",
+ " )\n",
+ "\n",
+ " # Tool was not invoked\n",
+ " elif not tool_output[\"parsed\"]:\n",
+ " print(\"Failed to invoke tool!\")\n",
+ " raise ValueError(\n",
+ " \"You did not use the provided tool! Be sure to invoke the tool to structure the output.\"\n",
+ " )\n",
+ " return tool_output\n",
+ "\n",
+ "\n",
+ "# Chain with output check\n",
+ "code_chain_claude_raw = (\n",
+ " code_gen_prompt_claude | structured_llm_claude | check_claude_output\n",
+ ")\n",
+ "\n",
+ "\n",
+ "def insert_errors(inputs):\n",
+ " \"\"\"Insert errors for tool parsing in the messages\"\"\"\n",
+ "\n",
+ " # Get errors\n",
+ " error = inputs[\"error\"]\n",
+ " messages = inputs[\"messages\"]\n",
+ " messages += [\n",
+ " (\n",
+ " \"assistant\",\n",
+ " f\"Retry. You are required to fix the parsing errors: {error} \\n\\n You must invoke the provided tool.\",\n",
+ " )\n",
+ " ]\n",
+ " return {\n",
+ " \"messages\": messages,\n",
+ " \"context\": inputs[\"context\"],\n",
+ " }\n",
+ "\n",
+ "\n",
+ "# This will be run as a fallback chain\n",
+ "fallback_chain = insert_errors | code_chain_claude_raw\n",
+ "N = 3 # Max re-tries\n",
+ "code_gen_chain_re_try = code_chain_claude_raw.with_fallbacks(\n",
+ " fallbacks=[fallback_chain] * N, exception_key=\"error\"\n",
+ ")\n",
+ "\n",
+ "\n",
+ "def parse_output(solution):\n",
+ " \"\"\"When we add 'include_raw=True' to structured output,\n",
+ " it will return a dict w 'raw', 'parsed', 'parsing_error'.\"\"\"\n",
+ "\n",
+ " return solution[\"parsed\"]\n",
+ "\n",
+ "\n",
+ "# Optional: With re-try to correct for failure to invoke tool\n",
+ "code_gen_chain = code_gen_chain_re_try | parse_output\n",
+ "\n",
+ "# No re-try\n",
+ "code_gen_chain = code_gen_prompt_claude | structured_llm_claude | parse_output"
+ ]
},
{
"cell_type": "code",
@@ -92,7 +260,14 @@
"scrolled": true
},
"outputs": [],
- "source": ["# Test\nquestion = \"How do I build a RAG chain in LCEL?\"\nsolution = code_gen_chain.invoke(\n {\"context\": concatenated_content, \"messages\": [(\"user\", question)]}\n)\nsolution"]
+ "source": [
+ "# Test\n",
+ "question = \"How do I build a RAG chain in LCEL?\"\n",
+ "solution = code_gen_chain.invoke(\n",
+ " {\"context\": concatenated_content, \"messages\": [(\"user\", question)]}\n",
+ ")\n",
+ "solution"
+ ]
},
{
"cell_type": "markdown",
@@ -110,7 +285,26 @@
"id": "c185f1a2-e943-4bed-b833-4243c9c64092",
"metadata": {},
"outputs": [],
- "source": ["from typing import List, TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n error : Binary flag for control flow to indicate whether test error was tripped\n messages : With user question, error messages, reasoning\n generation : Code solution\n iterations : Number of tries\n \"\"\"\n\n error: str\n messages: List\n generation: str\n iterations: int"]
+ "source": [
+ "from typing import List, TypedDict\n",
+ "\n",
+ "\n",
+ "class GraphState(TypedDict):\n",
+ " \"\"\"\n",
+ " Represents the state of our graph.\n",
+ "\n",
+ " Attributes:\n",
+ " error : Binary flag for control flow to indicate whether test error was tripped\n",
+ " messages : With user question, error messages, reasoning\n",
+ " generation : Code solution\n",
+ " iterations : Number of tries\n",
+ " \"\"\"\n",
+ "\n",
+ " error: str\n",
+ " messages: List\n",
+ " generation: str\n",
+ " iterations: int"
+ ]
},
{
"cell_type": "markdown",
@@ -128,7 +322,177 @@
"id": "b70e8301-63ae-4f7e-ad8f-c9a052fe3566",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.pydantic_v1 import BaseModel, Field\n\n### Parameter\n\n# Max tries\nmax_iterations = 3\n# Reflect\n# flag = 'reflect'\nflag = \"do not reflect\"\n\n### Nodes\n\n\ndef generate(state: GraphState):\n \"\"\"\n Generate a code solution\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation\n \"\"\"\n\n print(\"---GENERATING CODE SOLUTION---\")\n\n # State\n messages = state[\"messages\"]\n iterations = state[\"iterations\"]\n error = state[\"error\"]\n\n # We have been routed back to generation with an error\n if error == \"yes\":\n messages += [\n (\n \"user\",\n \"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\",\n )\n ]\n\n # Solution\n code_solution = code_gen_chain.invoke(\n {\"context\": concatenated_content, \"messages\": messages}\n )\n messages += [\n (\n \"assistant\",\n f\"{code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n )\n ]\n\n # Increment\n iterations = iterations + 1\n return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n\n\ndef code_check(state: GraphState):\n \"\"\"\n Check code\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, error\n \"\"\"\n\n print(\"---CHECKING CODE---\")\n\n # State\n messages = state[\"messages\"]\n code_solution = state[\"generation\"]\n iterations = state[\"iterations\"]\n\n # Get solution components\n imports = code_solution.imports\n code = code_solution.code\n\n # Check imports\n try:\n exec(imports)\n except Exception as e:\n print(\"---CODE IMPORT CHECK: FAILED---\")\n error_message = [(\"user\", f\"Your solution failed the import test: {e}\")]\n messages += error_message\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"yes\",\n }\n\n # Check execution\n try:\n exec(imports + \"\\n\" + code)\n except Exception as e:\n print(\"---CODE BLOCK CHECK: FAILED---\")\n error_message = [(\"user\", f\"Your solution failed the code execution test: {e}\")]\n messages += error_message\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"yes\",\n }\n\n # No errors\n print(\"---NO CODE TEST FAILURES---\")\n return {\n \"generation\": code_solution,\n \"messages\": messages,\n \"iterations\": iterations,\n \"error\": \"no\",\n }\n\n\ndef reflect(state: GraphState):\n \"\"\"\n Reflect on errors\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation\n \"\"\"\n\n print(\"---GENERATING CODE SOLUTION---\")\n\n # State\n messages = state[\"messages\"]\n iterations = state[\"iterations\"]\n code_solution = state[\"generation\"]\n\n # Prompt reflection\n\n # Add reflection\n reflections = code_gen_chain.invoke(\n {\"context\": concatenated_content, \"messages\": messages}\n )\n messages += [(\"assistant\", f\"Here are reflections on the error: {reflections}\")]\n return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n\n\n### Edges\n\n\ndef decide_to_finish(state: GraphState):\n \"\"\"\n Determines whether to finish.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Next node to call\n \"\"\"\n error = state[\"error\"]\n iterations = state[\"iterations\"]\n\n if error == \"no\" or iterations == max_iterations:\n print(\"---DECISION: FINISH---\")\n return \"end\"\n else:\n print(\"---DECISION: RE-TRY SOLUTION---\")\n if flag == \"reflect\":\n return \"reflect\"\n else:\n return \"generate\""]
+ "source": [
+ "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+ "\n",
+ "### Parameter\n",
+ "\n",
+ "# Max tries\n",
+ "max_iterations = 3\n",
+ "# Reflect\n",
+ "# flag = 'reflect'\n",
+ "flag = \"do not reflect\"\n",
+ "\n",
+ "### Nodes\n",
+ "\n",
+ "\n",
+ "def generate(state: GraphState):\n",
+ " \"\"\"\n",
+ " Generate a code solution\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, generation\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---GENERATING CODE SOLUTION---\")\n",
+ "\n",
+ " # State\n",
+ " messages = state[\"messages\"]\n",
+ " iterations = state[\"iterations\"]\n",
+ " error = state[\"error\"]\n",
+ "\n",
+ " # We have been routed back to generation with an error\n",
+ " if error == \"yes\":\n",
+ " messages += [\n",
+ " (\n",
+ " \"user\",\n",
+ " \"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\",\n",
+ " )\n",
+ " ]\n",
+ "\n",
+ " # Solution\n",
+ " code_solution = code_gen_chain.invoke(\n",
+ " {\"context\": concatenated_content, \"messages\": messages}\n",
+ " )\n",
+ " messages += [\n",
+ " (\n",
+ " \"assistant\",\n",
+ " f\"{code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n",
+ " )\n",
+ " ]\n",
+ "\n",
+ " # Increment\n",
+ " iterations = iterations + 1\n",
+ " return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
+ "\n",
+ "\n",
+ "def code_check(state: GraphState):\n",
+ " \"\"\"\n",
+ " Check code\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, error\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECKING CODE---\")\n",
+ "\n",
+ " # State\n",
+ " messages = state[\"messages\"]\n",
+ " code_solution = state[\"generation\"]\n",
+ " iterations = state[\"iterations\"]\n",
+ "\n",
+ " # Get solution components\n",
+ " imports = code_solution.imports\n",
+ " code = code_solution.code\n",
+ "\n",
+ " # Check imports\n",
+ " try:\n",
+ " exec(imports)\n",
+ " except Exception as e:\n",
+ " print(\"---CODE IMPORT CHECK: FAILED---\")\n",
+ " error_message = [(\"user\", f\"Your solution failed the import test: {e}\")]\n",
+ " messages += error_message\n",
+ " return {\n",
+ " \"generation\": code_solution,\n",
+ " \"messages\": messages,\n",
+ " \"iterations\": iterations,\n",
+ " \"error\": \"yes\",\n",
+ " }\n",
+ "\n",
+ " # Check execution\n",
+ " try:\n",
+ " exec(imports + \"\\n\" + code)\n",
+ " except Exception as e:\n",
+ " print(\"---CODE BLOCK CHECK: FAILED---\")\n",
+ " error_message = [(\"user\", f\"Your solution failed the code execution test: {e}\")]\n",
+ " messages += error_message\n",
+ " return {\n",
+ " \"generation\": code_solution,\n",
+ " \"messages\": messages,\n",
+ " \"iterations\": iterations,\n",
+ " \"error\": \"yes\",\n",
+ " }\n",
+ "\n",
+ " # No errors\n",
+ " print(\"---NO CODE TEST FAILURES---\")\n",
+ " return {\n",
+ " \"generation\": code_solution,\n",
+ " \"messages\": messages,\n",
+ " \"iterations\": iterations,\n",
+ " \"error\": \"no\",\n",
+ " }\n",
+ "\n",
+ "\n",
+ "def reflect(state: GraphState):\n",
+ " \"\"\"\n",
+ " Reflect on errors\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, generation\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---GENERATING CODE SOLUTION---\")\n",
+ "\n",
+ " # State\n",
+ " messages = state[\"messages\"]\n",
+ " iterations = state[\"iterations\"]\n",
+ " code_solution = state[\"generation\"]\n",
+ "\n",
+ " # Prompt reflection\n",
+ "\n",
+ " # Add reflection\n",
+ " reflections = code_gen_chain.invoke(\n",
+ " {\"context\": concatenated_content, \"messages\": messages}\n",
+ " )\n",
+ " messages += [(\"assistant\", f\"Here are reflections on the error: {reflections}\")]\n",
+ " return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n",
+ "\n",
+ "\n",
+ "### Edges\n",
+ "\n",
+ "\n",
+ "def decide_to_finish(state: GraphState):\n",
+ " \"\"\"\n",
+ " Determines whether to finish.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Next node to call\n",
+ " \"\"\"\n",
+ " error = state[\"error\"]\n",
+ " iterations = state[\"iterations\"]\n",
+ "\n",
+ " if error == \"no\" or iterations == max_iterations:\n",
+ " print(\"---DECISION: FINISH---\")\n",
+ " return \"end\"\n",
+ " else:\n",
+ " print(\"---DECISION: RE-TRY SOLUTION---\")\n",
+ " if flag == \"reflect\":\n",
+ " return \"reflect\"\n",
+ " else:\n",
+ " return \"generate\""
+ ]
},
{
"cell_type": "code",
@@ -136,7 +500,31 @@
"id": "f66b4e00-4731-42c8-bc38-72dd0ff7c92c",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"generate\", generate) # generation solution\nworkflow.add_node(\"check_code\", code_check) # check code\nworkflow.add_node(\"reflect\", reflect) # reflect\n\n# Build graph\nworkflow.add_edge(START, \"generate\")\nworkflow.add_edge(\"generate\", \"check_code\")\nworkflow.add_conditional_edges(\n \"check_code\",\n decide_to_finish,\n {\n \"end\": END,\n \"reflect\": \"reflect\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"reflect\", \"generate\")\napp = workflow.compile()"]
+ "source": [
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "workflow = StateGraph(GraphState)\n",
+ "\n",
+ "# Define the nodes\n",
+ "workflow.add_node(\"generate\", generate) # generation solution\n",
+ "workflow.add_node(\"check_code\", code_check) # check code\n",
+ "workflow.add_node(\"reflect\", reflect) # reflect\n",
+ "\n",
+ "# Build graph\n",
+ "workflow.add_edge(START, \"generate\")\n",
+ "workflow.add_edge(\"generate\", \"check_code\")\n",
+ "workflow.add_conditional_edges(\n",
+ " \"check_code\",\n",
+ " decide_to_finish,\n",
+ " {\n",
+ " \"end\": END,\n",
+ " \"reflect\": \"reflect\",\n",
+ " \"generate\": \"generate\",\n",
+ " },\n",
+ ")\n",
+ "workflow.add_edge(\"reflect\", \"generate\")\n",
+ "app = workflow.compile()"
+ ]
},
{
"cell_type": "code",
@@ -144,7 +532,10 @@
"id": "9bcaafe4-ddcf-4fab-8620-2d9b6c508f98",
"metadata": {},
"outputs": [],
- "source": ["question = \"How can I directly pass a string to a runnable and use it to construct the input needed for my prompt?\"\napp.invoke({\"messages\": [(\"user\", question)], \"iterations\": 0})"]
+ "source": [
+ "question = \"How can I directly pass a string to a runnable and use it to construct the input needed for my prompt?\"\n",
+ "app.invoke({\"messages\": [(\"user\", question)], \"iterations\": 0})"
+ ]
},
{
"cell_type": "markdown",
@@ -172,7 +563,11 @@
"id": "678e8954-56b5-4cc6-be26-f7f2a060b242",
"metadata": {},
"outputs": [],
- "source": ["import langsmith\n\nclient = langsmith.Client()"]
+ "source": [
+ "import langsmith\n",
+ "\n",
+ "client = langsmith.Client()"
+ ]
},
{
"cell_type": "code",
@@ -180,7 +575,13 @@
"id": "ef7cf662-7a6f-4dee-965c-6309d4045feb",
"metadata": {},
"outputs": [],
- "source": ["# Clone the dataset to your tenant to use it\npublic_dataset = (\n \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n)\nclient.clone_public_dataset(public_dataset)"]
+ "source": [
+ "# Clone the dataset to your tenant to use it\n",
+ "public_dataset = (\n",
+ " \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n",
+ ")\n",
+ "client.clone_public_dataset(public_dataset)"
+ ]
},
{
"cell_type": "markdown",
@@ -196,7 +597,28 @@
"id": "455a34ea-52cb-4ae5-9f4a-7e4a08cd0c09",
"metadata": {},
"outputs": [],
- "source": ["from langsmith.schemas import Example, Run\n\n\ndef check_import(run: Run, example: Example) -> dict:\n imports = run.outputs.get(\"imports\")\n try:\n exec(imports)\n return {\"key\": \"import_check\", \"score\": 1}\n except Exception:\n return {\"key\": \"import_check\", \"score\": 0}\n\n\ndef check_execution(run: Run, example: Example) -> dict:\n imports = run.outputs.get(\"imports\")\n code = run.outputs.get(\"code\")\n try:\n exec(imports + \"\\n\" + code)\n return {\"key\": \"code_execution_check\", \"score\": 1}\n except Exception:\n return {\"key\": \"code_execution_check\", \"score\": 0}"]
+ "source": [
+ "from langsmith.schemas import Example, Run\n",
+ "\n",
+ "\n",
+ "def check_import(run: Run, example: Example) -> dict:\n",
+ " imports = run.outputs.get(\"imports\")\n",
+ " try:\n",
+ " exec(imports)\n",
+ " return {\"key\": \"import_check\", \"score\": 1}\n",
+ " except Exception:\n",
+ " return {\"key\": \"import_check\", \"score\": 0}\n",
+ "\n",
+ "\n",
+ "def check_execution(run: Run, example: Example) -> dict:\n",
+ " imports = run.outputs.get(\"imports\")\n",
+ " code = run.outputs.get(\"code\")\n",
+ " try:\n",
+ " exec(imports + \"\\n\" + code)\n",
+ " return {\"key\": \"code_execution_check\", \"score\": 1}\n",
+ " except Exception:\n",
+ " return {\"key\": \"code_execution_check\", \"score\": 0}"
+ ]
},
{
"cell_type": "markdown",
@@ -212,7 +634,22 @@
"id": "c8fa6bcb-b245-4422-b79a-582cd8a7d7ea",
"metadata": {},
"outputs": [],
- "source": ["def predict_base_case(example: dict):\n \"\"\"Context stuffing\"\"\"\n solution = code_gen_chain.invoke(\n {\"context\": concatenated_content, \"messages\": [(\"user\", example[\"question\"])]}\n )\n solution_structured = code_gen_chain.invoke([(\"code\", solution)])\n return {\"imports\": solution_structured.imports, \"code\": solution_structured.code}\n\n\ndef predict_langgraph(example: dict):\n \"\"\"LangGraph\"\"\"\n graph = app.invoke({\"messages\": [(\"user\", example[\"question\"])], \"iterations\": 0})\n solution = graph[\"generation\"]\n return {\"imports\": solution.imports, \"code\": solution.code}"]
+ "source": [
+ "def predict_base_case(example: dict):\n",
+ " \"\"\"Context stuffing\"\"\"\n",
+ " solution = code_gen_chain.invoke(\n",
+ " {\"context\": concatenated_content, \"messages\": [(\"user\", example[\"question\"])]}\n",
+ " )\n",
+ " solution_structured = code_gen_chain.invoke([(\"code\", solution)])\n",
+ " return {\"imports\": solution_structured.imports, \"code\": solution_structured.code}\n",
+ "\n",
+ "\n",
+ "def predict_langgraph(example: dict):\n",
+ " \"\"\"LangGraph\"\"\"\n",
+ " graph = app.invoke({\"messages\": [(\"user\", example[\"question\"])], \"iterations\": 0})\n",
+ " solution = graph[\"generation\"]\n",
+ " return {\"imports\": solution.imports, \"code\": solution.code}"
+ ]
},
{
"cell_type": "code",
@@ -220,7 +657,15 @@
"id": "d9c57468-97f6-47d6-a5e9-c09b53bfdd83",
"metadata": {},
"outputs": [],
- "source": ["from langsmith.evaluation import evaluate\n\n# Evaluator\ncode_evalulator = [check_import, check_execution]\n\n# Dataset\ndataset_name = \"test-LCEL-code-gen\""]
+ "source": [
+ "from langsmith.evaluation import evaluate\n",
+ "\n",
+ "# Evaluator\n",
+ "code_evalulator = [check_import, check_execution]\n",
+ "\n",
+ "# Dataset\n",
+ "dataset_name = \"test-LCEL-code-gen\""
+ ]
},
{
"cell_type": "code",
@@ -228,7 +673,19 @@
"id": "2dacccf0-d73f-4017-aaf0-9806ffe5bd2c",
"metadata": {},
"outputs": [],
- "source": ["# Run base case\nexperiment_results_ = evaluate(\n predict_base_case,\n data=dataset_name,\n evaluators=code_evalulator,\n experiment_prefix=f\"test-without-langgraph-{expt_llm}\",\n max_concurrency=2,\n metadata={\n \"llm\": expt_llm,\n },\n)"]
+ "source": [
+ "# Run base case\n",
+ "experiment_results_ = evaluate(\n",
+ " predict_base_case,\n",
+ " data=dataset_name,\n",
+ " evaluators=code_evalulator,\n",
+ " experiment_prefix=f\"test-without-langgraph-{expt_llm}\",\n",
+ " max_concurrency=2,\n",
+ " metadata={\n",
+ " \"llm\": expt_llm,\n",
+ " },\n",
+ ")"
+ ]
},
{
"cell_type": "code",
@@ -236,7 +693,20 @@
"id": "71d90f9e-9dad-410c-a709-093d275029ae",
"metadata": {},
"outputs": [],
- "source": ["# Run with langgraph\nexperiment_results = evaluate(\n predict_langgraph,\n data=dataset_name,\n evaluators=code_evalulator,\n experiment_prefix=f\"test-with-langgraph-{expt_llm}-{flag}\",\n max_concurrency=2,\n metadata={\n \"llm\": expt_llm,\n \"feedback\": flag,\n },\n)"]
+ "source": [
+ "# Run with langgraph\n",
+ "experiment_results = evaluate(\n",
+ " predict_langgraph,\n",
+ " data=dataset_name,\n",
+ " evaluators=code_evalulator,\n",
+ " experiment_prefix=f\"test-with-langgraph-{expt_llm}-{flag}\",\n",
+ " max_concurrency=2,\n",
+ " metadata={\n",
+ " \"llm\": expt_llm,\n",
+ " \"feedback\": flag,\n",
+ " },\n",
+ ")"
+ ]
},
{
"cell_type": "markdown",
@@ -251,14 +721,6 @@
"\n",
"https://smith.langchain.com/public/78a3d858-c811-4e46-91cb-0f10ef56260b/d"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a42333c3-c098-4576-ae2a-0258de64ece2",
- "metadata": {},
- "outputs": [],
- "source": [""]
}
],
"metadata": {
diff --git a/examples/human_in_the_loop/time-travel.ipynb b/examples/human_in_the_loop/time-travel.ipynb
index 765a1b221..74cb774c3 100644
--- a/examples/human_in_the_loop/time-travel.ipynb
+++ b/examples/human_in_the_loop/time-travel.ipynb
@@ -40,7 +40,8 @@
"metadata": {},
"outputs": [],
"source": [
- "%%capture --no-stderr\n%pip install --quiet -U langgraph langchain_anthropic"
+ "%%capture --no-stderr\n",
+ "%pip install --quiet -U langgraph langchain_anthropic"
]
},
{
@@ -58,7 +59,7 @@
"metadata": {},
"outputs": [
{
- "name": "stdin",
+ "name": "stdout",
"output_type": "stream",
"text": [
"ANTHROPIC_API_KEY: ········\n"
@@ -66,7 +67,16 @@
}
],
"source": [
- "import getpass\nimport os\n\n\ndef _set_env(var: str):\n if not os.environ.get(var):\n os.environ[var] = getpass.getpass(f\"{var}: \")\n\n\n_set_env(\"ANTHROPIC_API_KEY\")"
+ "import getpass\n",
+ "import os\n",
+ "\n",
+ "\n",
+ "def _set_env(var: str):\n",
+ " if not os.environ.get(var):\n",
+ " os.environ[var] = getpass.getpass(f\"{var}: \")\n",
+ "\n",
+ "\n",
+ "_set_env(\"ANTHROPIC_API_KEY\")"
]
},
{
@@ -84,7 +94,8 @@
"metadata": {},
"outputs": [],
"source": [
- "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n_set_env(\"LANGCHAIN_API_KEY\")"
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "_set_env(\"LANGCHAIN_API_KEY\")"
]
},
{
@@ -256,7 +267,12 @@
}
],
"source": [
- "from langchain_core.messages import HumanMessage\n\nconfig = {\"configurable\": {\"thread_id\": \"1\"}}\ninput_message = HumanMessage(content=\"Use the search tool to look up the weather in SF\")\nfor event in app.stream({\"messages\": [input_message]}, config, stream_mode=\"values\"):\n event[\"messages\"][-1].pretty_print()"
+ "from langchain_core.messages import HumanMessage\n",
+ "\n",
+ "config = {\"configurable\": {\"thread_id\": \"1\"}}\n",
+ "input_message = HumanMessage(content=\"Use the search tool to look up the weather in SF\")\n",
+ "for event in app.stream({\"messages\": [input_message]}, config, stream_mode=\"values\"):\n",
+ " event[\"messages\"][-1].pretty_print()"
]
},
{
@@ -293,7 +309,11 @@
}
],
"source": [
- "all_states = []\nfor state in app.get_state_history(config):\n print(state)\n all_states.append(state)\n print(\"--\")"
+ "all_states = []\n",
+ "for state in app.get_state_history(config):\n",
+ " print(state)\n",
+ " all_states.append(state)\n",
+ " print(\"--\")"
]
},
{
@@ -383,7 +403,9 @@
}
],
"source": [
- "for event in app.stream(None, to_replay.config):\n for v in event.values():\n print(v)"
+ "for event in app.stream(None, to_replay.config):\n",
+ " for v in event.values():\n",
+ " print(v)"
]
},
{
@@ -442,7 +464,9 @@
}
],
"source": [
- "for event in app.stream(None, branch_config):\n for v in event.values():\n print(v)"
+ "for event in app.stream(None, branch_config):\n",
+ " for v in event.values():\n",
+ " print(v)"
]
},
{
@@ -535,16 +559,6 @@
"source": [
"You can see the snapshot was updated and now correctly reflects that there is no next step."
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "74a7a5ed-0c14-4883-a16b-d70aaf40f7ea",
- "metadata": {},
- "outputs": [],
- "source": [
- ""
- ]
}
],
"metadata": {
diff --git a/examples/multi_agent/multi-agent-collaboration.ipynb b/examples/multi_agent/multi-agent-collaboration.ipynb
index c660fd296..ff06aec19 100644
--- a/examples/multi_agent/multi-agent-collaboration.ipynb
+++ b/examples/multi_agent/multi-agent-collaboration.ipynb
@@ -26,7 +26,10 @@
"id": "0d7b6dcc-c985-46e2-8457-7e6b0298b950",
"metadata": {},
"outputs": [],
- "source": ["%%capture --no-stderr\n%pip install -U langchain langchain_openai langsmith pandas langchain_experimental matplotlib langgraph langchain_core"]
+ "source": [
+ "%%capture --no-stderr\n",
+ "%pip install -U langchain langchain_openai langsmith pandas langchain_experimental matplotlib langgraph langchain_core"
+ ]
},
{
"cell_type": "code",
@@ -34,7 +37,24 @@
"id": "743c19df-6da9-4d1e-b2d2-ea40080b9fdc",
"metadata": {},
"outputs": [],
- "source": ["import getpass\nimport os\n\n\ndef _set_if_undefined(var: str):\n if not os.environ.get(var):\n os.environ[var] = getpass.getpass(f\"Please provide your {var}\")\n\n\n_set_if_undefined(\"OPENAI_API_KEY\")\n_set_if_undefined(\"LANGCHAIN_API_KEY\")\n_set_if_undefined(\"TAVILY_API_KEY\")\n\n# Optional, add tracing in LangSmith\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_PROJECT\"] = \"Multi-agent Collaboration\""]
+ "source": [
+ "import getpass\n",
+ "import os\n",
+ "\n",
+ "\n",
+ "def _set_if_undefined(var: str):\n",
+ " if not os.environ.get(var):\n",
+ " os.environ[var] = getpass.getpass(f\"Please provide your {var}\")\n",
+ "\n",
+ "\n",
+ "_set_if_undefined(\"OPENAI_API_KEY\")\n",
+ "_set_if_undefined(\"LANGCHAIN_API_KEY\")\n",
+ "_set_if_undefined(\"TAVILY_API_KEY\")\n",
+ "\n",
+ "# Optional, add tracing in LangSmith\n",
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "os.environ[\"LANGCHAIN_PROJECT\"] = \"Multi-agent Collaboration\""
+ ]
},
{
"cell_type": "markdown",
@@ -54,7 +74,38 @@
"id": "4325a10e-38dc-4a98-9004-e1525eaba377",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.messages import (\n BaseMessage,\n HumanMessage,\n ToolMessage,\n)\nfrom langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n\nfrom langgraph.graph import END, StateGraph, START\n\n\ndef create_agent(llm, tools, system_message: str):\n \"\"\"Create an agent.\"\"\"\n prompt = ChatPromptTemplate.from_messages(\n [\n (\n \"system\",\n \"You are a helpful AI assistant, collaborating with other assistants.\"\n \" Use the provided tools to progress towards answering the question.\"\n \" If you are unable to fully answer, that's OK, another assistant with different tools \"\n \" will help where you left off. Execute what you can to make progress.\"\n \" If you or any of the other assistants have the final answer or deliverable,\"\n \" prefix your response with FINAL ANSWER so the team knows to stop.\"\n \" You have access to the following tools: {tool_names}.\\n{system_message}\",\n ),\n MessagesPlaceholder(variable_name=\"messages\"),\n ]\n )\n prompt = prompt.partial(system_message=system_message)\n prompt = prompt.partial(tool_names=\", \".join([tool.name for tool in tools]))\n return prompt | llm.bind_tools(tools)"]
+ "source": [
+ "from langchain_core.messages import (\n",
+ " BaseMessage,\n",
+ " HumanMessage,\n",
+ " ToolMessage,\n",
+ ")\n",
+ "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
+ "\n",
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "\n",
+ "def create_agent(llm, tools, system_message: str):\n",
+ " \"\"\"Create an agent.\"\"\"\n",
+ " prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\n",
+ " \"system\",\n",
+ " \"You are a helpful AI assistant, collaborating with other assistants.\"\n",
+ " \" Use the provided tools to progress towards answering the question.\"\n",
+ " \" If you are unable to fully answer, that's OK, another assistant with different tools \"\n",
+ " \" will help where you left off. Execute what you can to make progress.\"\n",
+ " \" If you or any of the other assistants have the final answer or deliverable,\"\n",
+ " \" prefix your response with FINAL ANSWER so the team knows to stop.\"\n",
+ " \" You have access to the following tools: {tool_names}.\\n{system_message}\",\n",
+ " ),\n",
+ " MessagesPlaceholder(variable_name=\"messages\"),\n",
+ " ]\n",
+ " )\n",
+ " prompt = prompt.partial(system_message=system_message)\n",
+ " prompt = prompt.partial(tool_names=\", \".join([tool.name for tool in tools]))\n",
+ " return prompt | llm.bind_tools(tools)"
+ ]
},
{
"cell_type": "markdown",
@@ -72,7 +123,35 @@
"id": "ca076f3b-a729-4ca9-8f91-05c2ba58d610",
"metadata": {},
"outputs": [],
- "source": ["from typing import Annotated\n\nfrom langchain_community.tools.tavily_search import TavilySearchResults\nfrom langchain_core.tools import tool\nfrom langchain_experimental.utilities import PythonREPL\n\ntavily_tool = TavilySearchResults(max_results=5)\n\n# Warning: This executes code locally, which can be unsafe when not sandboxed\n\nrepl = PythonREPL()\n\n\n@tool\ndef python_repl(\n code: Annotated[str, \"The python code to execute to generate your chart.\"],\n):\n \"\"\"Use this to execute python code. If you want to see the output of a value,\n you should print it out with `print(...)`. This is visible to the user.\"\"\"\n try:\n result = repl.run(code)\n except BaseException as e:\n return f\"Failed to execute. Error: {repr(e)}\"\n result_str = f\"Successfully executed:\\n```python\\n{code}\\n```\\nStdout: {result}\"\n return (\n result_str + \"\\n\\nIf you have completed all tasks, respond with FINAL ANSWER.\"\n )"]
+ "source": [
+ "from typing import Annotated\n",
+ "\n",
+ "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+ "from langchain_core.tools import tool\n",
+ "from langchain_experimental.utilities import PythonREPL\n",
+ "\n",
+ "tavily_tool = TavilySearchResults(max_results=5)\n",
+ "\n",
+ "# Warning: This executes code locally, which can be unsafe when not sandboxed\n",
+ "\n",
+ "repl = PythonREPL()\n",
+ "\n",
+ "\n",
+ "@tool\n",
+ "def python_repl(\n",
+ " code: Annotated[str, \"The python code to execute to generate your chart.\"],\n",
+ "):\n",
+ " \"\"\"Use this to execute python code. If you want to see the output of a value,\n",
+ " you should print it out with `print(...)`. This is visible to the user.\"\"\"\n",
+ " try:\n",
+ " result = repl.run(code)\n",
+ " except BaseException as e:\n",
+ " return f\"Failed to execute. Error: {repr(e)}\"\n",
+ " result_str = f\"Successfully executed:\\n```python\\n{code}\\n```\\nStdout: {result}\"\n",
+ " return (\n",
+ " result_str + \"\\n\\nIf you have completed all tasks, respond with FINAL ANSWER.\"\n",
+ " )"
+ ]
},
{
"cell_type": "markdown",
@@ -100,7 +179,19 @@
"id": "290c91d4-f6f4-443c-8181-233d39102974",
"metadata": {},
"outputs": [],
- "source": ["import operator\nfrom typing import Annotated, Sequence, TypedDict\n\nfrom langchain_openai import ChatOpenAI\n\n\n# This defines the object that is passed between each node\n# in the graph. We will create different nodes for each agent and tool\nclass AgentState(TypedDict):\n messages: Annotated[Sequence[BaseMessage], operator.add]\n sender: str"]
+ "source": [
+ "import operator\n",
+ "from typing import Annotated, Sequence, TypedDict\n",
+ "\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "\n",
+ "# This defines the object that is passed between each node\n",
+ "# in the graph. We will create different nodes for each agent and tool\n",
+ "class AgentState(TypedDict):\n",
+ " messages: Annotated[Sequence[BaseMessage], operator.add]\n",
+ " sender: str"
+ ]
},
{
"cell_type": "markdown",
@@ -118,7 +209,46 @@
"id": "71b790ca-9cef-4b22-b469-4b1d5d8424d6",
"metadata": {},
"outputs": [],
- "source": ["import functools\n\nfrom langchain_core.messages import AIMessage\n\n\n# Helper function to create a node for a given agent\ndef agent_node(state, agent, name):\n result = agent.invoke(state)\n # We convert the agent output into a format that is suitable to append to the global state\n if isinstance(result, ToolMessage):\n pass\n else:\n result = AIMessage(**result.dict(exclude={\"type\", \"name\"}), name=name)\n return {\n \"messages\": [result],\n # Since we have a strict workflow, we can\n # track the sender so we know who to pass to next.\n \"sender\": name,\n }\n\n\nllm = ChatOpenAI(model=\"gpt-4o\")\n\n# Research agent and node\nresearch_agent = create_agent(\n llm,\n [tavily_tool],\n system_message=\"You should provide accurate data for the chart_generator to use.\",\n)\nresearch_node = functools.partial(agent_node, agent=research_agent, name=\"Researcher\")\n\n# chart_generator\nchart_agent = create_agent(\n llm,\n [python_repl],\n system_message=\"Any charts you display will be visible by the user.\",\n)\nchart_node = functools.partial(agent_node, agent=chart_agent, name=\"chart_generator\")"]
+ "source": [
+ "import functools\n",
+ "\n",
+ "from langchain_core.messages import AIMessage\n",
+ "\n",
+ "\n",
+ "# Helper function to create a node for a given agent\n",
+ "def agent_node(state, agent, name):\n",
+ " result = agent.invoke(state)\n",
+ " # We convert the agent output into a format that is suitable to append to the global state\n",
+ " if isinstance(result, ToolMessage):\n",
+ " pass\n",
+ " else:\n",
+ " result = AIMessage(**result.dict(exclude={\"type\", \"name\"}), name=name)\n",
+ " return {\n",
+ " \"messages\": [result],\n",
+ " # Since we have a strict workflow, we can\n",
+ " # track the sender so we know who to pass to next.\n",
+ " \"sender\": name,\n",
+ " }\n",
+ "\n",
+ "\n",
+ "llm = ChatOpenAI(model=\"gpt-4o\")\n",
+ "\n",
+ "# Research agent and node\n",
+ "research_agent = create_agent(\n",
+ " llm,\n",
+ " [tavily_tool],\n",
+ " system_message=\"You should provide accurate data for the chart_generator to use.\",\n",
+ ")\n",
+ "research_node = functools.partial(agent_node, agent=research_agent, name=\"Researcher\")\n",
+ "\n",
+ "# chart_generator\n",
+ "chart_agent = create_agent(\n",
+ " llm,\n",
+ " [python_repl],\n",
+ " system_message=\"Any charts you display will be visible by the user.\",\n",
+ ")\n",
+ "chart_node = functools.partial(agent_node, agent=chart_agent, name=\"chart_generator\")"
+ ]
},
{
"cell_type": "markdown",
@@ -136,7 +266,12 @@
"id": "d9a79c76-5c7c-42f6-91cf-635bc8305804",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.prebuilt import ToolNode\n\ntools = [tavily_tool, python_repl]\ntool_node = ToolNode(tools)"]
+ "source": [
+ "from langgraph.prebuilt import ToolNode\n",
+ "\n",
+ "tools = [tavily_tool, python_repl]\n",
+ "tool_node = ToolNode(tools)"
+ ]
},
{
"cell_type": "markdown",
@@ -154,7 +289,23 @@
"id": "4f4b4d37-e8a3-4abb-8d42-eaea26016f35",
"metadata": {},
"outputs": [],
- "source": ["# Either agent can decide to end\nfrom typing import Literal\n\n\ndef router(state) -> Literal[\"call_tool\", \"__end__\", \"continue\"]:\n # This is the router\n messages = state[\"messages\"]\n last_message = messages[-1]\n if last_message.tool_calls:\n # The previous agent is invoking a tool\n return \"call_tool\"\n if \"FINAL ANSWER\" in last_message.content:\n # Any agent decided the work is done\n return \"__end__\"\n return \"continue\""]
+ "source": [
+ "# Either agent can decide to end\n",
+ "from typing import Literal\n",
+ "\n",
+ "\n",
+ "def router(state) -> Literal[\"call_tool\", \"__end__\", \"continue\"]:\n",
+ " # This is the router\n",
+ " messages = state[\"messages\"]\n",
+ " last_message = messages[-1]\n",
+ " if last_message.tool_calls:\n",
+ " # The previous agent is invoking a tool\n",
+ " return \"call_tool\"\n",
+ " if \"FINAL ANSWER\" in last_message.content:\n",
+ " # Any agent decided the work is done\n",
+ " return \"__end__\"\n",
+ " return \"continue\""
+ ]
},
{
"cell_type": "markdown",
@@ -172,7 +323,39 @@
"id": "4dce3901-6ad5-4df5-8528-6e865cf96cb0",
"metadata": {},
"outputs": [],
- "source": ["workflow = StateGraph(AgentState)\n\nworkflow.add_node(\"Researcher\", research_node)\nworkflow.add_node(\"chart_generator\", chart_node)\nworkflow.add_node(\"call_tool\", tool_node)\n\nworkflow.add_conditional_edges(\n \"Researcher\",\n router,\n {\"continue\": \"chart_generator\", \"call_tool\": \"call_tool\", \"__end__\": END},\n)\nworkflow.add_conditional_edges(\n \"chart_generator\",\n router,\n {\"continue\": \"Researcher\", \"call_tool\": \"call_tool\", \"__end__\": END},\n)\n\nworkflow.add_conditional_edges(\n \"call_tool\",\n # Each agent node updates the 'sender' field\n # the tool calling node does not, meaning\n # this edge will route back to the original agent\n # who invoked the tool\n lambda x: x[\"sender\"],\n {\n \"Researcher\": \"Researcher\",\n \"chart_generator\": \"chart_generator\",\n },\n)\nworkflow.add_edge(START, \"Researcher\")\ngraph = workflow.compile()"]
+ "source": [
+ "workflow = StateGraph(AgentState)\n",
+ "\n",
+ "workflow.add_node(\"Researcher\", research_node)\n",
+ "workflow.add_node(\"chart_generator\", chart_node)\n",
+ "workflow.add_node(\"call_tool\", tool_node)\n",
+ "\n",
+ "workflow.add_conditional_edges(\n",
+ " \"Researcher\",\n",
+ " router,\n",
+ " {\"continue\": \"chart_generator\", \"call_tool\": \"call_tool\", \"__end__\": END},\n",
+ ")\n",
+ "workflow.add_conditional_edges(\n",
+ " \"chart_generator\",\n",
+ " router,\n",
+ " {\"continue\": \"Researcher\", \"call_tool\": \"call_tool\", \"__end__\": END},\n",
+ ")\n",
+ "\n",
+ "workflow.add_conditional_edges(\n",
+ " \"call_tool\",\n",
+ " # Each agent node updates the 'sender' field\n",
+ " # the tool calling node does not, meaning\n",
+ " # this edge will route back to the original agent\n",
+ " # who invoked the tool\n",
+ " lambda x: x[\"sender\"],\n",
+ " {\n",
+ " \"Researcher\": \"Researcher\",\n",
+ " \"chart_generator\": \"chart_generator\",\n",
+ " },\n",
+ ")\n",
+ "workflow.add_edge(START, \"Researcher\")\n",
+ "graph = workflow.compile()"
+ ]
},
{
"cell_type": "code",
@@ -191,7 +374,15 @@
"output_type": "display_data"
}
],
- "source": ["from IPython.display import Image, display\n\ntry:\n display(Image(graph.get_graph(xray=True).draw_mermaid_png()))\nexcept Exception:\n # This requires some extra dependencies and is optional\n pass"]
+ "source": [
+ "from IPython.display import Image, display\n",
+ "\n",
+ "try:\n",
+ " display(Image(graph.get_graph(xray=True).draw_mermaid_png()))\n",
+ "except Exception:\n",
+ " # This requires some extra dependencies and is optional\n",
+ " pass"
+ ]
},
{
"cell_type": "markdown",
@@ -290,15 +481,24 @@
]
}
],
- "source": ["events = graph.stream(\n {\n \"messages\": [\n HumanMessage(\n content=\"Fetch the UK's GDP over the past 5 years,\"\n \" then draw a line graph of it.\"\n \" Once you code it up, finish.\"\n )\n ],\n },\n # Maximum number of steps to take in the graph\n {\"recursion_limit\": 150},\n)\nfor s in events:\n print(s)\n print(\"----\")"]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "010fc36e-4116-4758-bcac-b02c7dcd405d",
- "metadata": {},
- "outputs": [],
- "source": [""]
+ "source": [
+ "events = graph.stream(\n",
+ " {\n",
+ " \"messages\": [\n",
+ " HumanMessage(\n",
+ " content=\"Fetch the UK's GDP over the past 5 years,\"\n",
+ " \" then draw a line graph of it.\"\n",
+ " \" Once you code it up, finish.\"\n",
+ " )\n",
+ " ],\n",
+ " },\n",
+ " # Maximum number of steps to take in the graph\n",
+ " {\"recursion_limit\": 150},\n",
+ ")\n",
+ "for s in events:\n",
+ " print(s)\n",
+ " print(\"----\")"
+ ]
}
],
"metadata": {
diff --git a/examples/plan-and-execute/plan-and-execute.ipynb b/examples/plan-and-execute/plan-and-execute.ipynb
index 95eb7d384..1f960d035 100644
--- a/examples/plan-and-execute/plan-and-execute.ipynb
+++ b/examples/plan-and-execute/plan-and-execute.ipynb
@@ -45,7 +45,10 @@
"id": "b451b58a-89bd-424f-8c06-0d9fe325e01b",
"metadata": {},
"outputs": [],
- "source": ["%%capture --no-stderr\n%pip install --quiet -U langgraph langchain-community langchain-openai tavily-python"]
+ "source": [
+ "%%capture --no-stderr\n",
+ "%pip install --quiet -U langgraph langchain-community langchain-openai tavily-python"
+ ]
},
{
"cell_type": "markdown",
@@ -61,7 +64,19 @@
"id": "ce438281-08d5-4804-afe7-e4089f7b016b",
"metadata": {},
"outputs": [],
- "source": ["import getpass\nimport os\n\n\ndef _set_env(var: str):\n if not os.environ.get(var):\n os.environ[var] = getpass.getpass(f\"{var}: \")\n\n\n_set_env(\"OPENAI_API_KEY\")\n_set_env(\"TAVILY_API_KEY\")"]
+ "source": [
+ "import getpass\n",
+ "import os\n",
+ "\n",
+ "\n",
+ "def _set_env(var: str):\n",
+ " if not os.environ.get(var):\n",
+ " os.environ[var] = getpass.getpass(f\"{var}: \")\n",
+ "\n",
+ "\n",
+ "_set_env(\"OPENAI_API_KEY\")\n",
+ "_set_env(\"TAVILY_API_KEY\")"
+ ]
},
{
"cell_type": "markdown",
@@ -77,7 +92,11 @@
"id": "01f460d1-f26f-47d1-ae76-de74d5d851de",
"metadata": {},
"outputs": [],
- "source": ["os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n_set_env(\"LANGCHAIN_API_KEY\")\nos.environ[\"LANGCHAIN_PROJECT\"] = \"Plan-and-execute\""]
+ "source": [
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "_set_env(\"LANGCHAIN_API_KEY\")\n",
+ "os.environ[\"LANGCHAIN_PROJECT\"] = \"Plan-and-execute\""
+ ]
},
{
"cell_type": "markdown",
@@ -95,7 +114,11 @@
"id": "25b9ec62-0675-4715-811c-9b32c635b22f",
"metadata": {},
"outputs": [],
- "source": ["from langchain_community.tools.tavily_search import TavilySearchResults\n\ntools = [TavilySearchResults(max_results=3)]"]
+ "source": [
+ "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+ "\n",
+ "tools = [TavilySearchResults(max_results=3)]"
+ ]
},
{
"cell_type": "markdown",
@@ -128,7 +151,20 @@
]
}
],
- "source": ["from langchain import hub\nfrom langchain_openai import ChatOpenAI\n\nfrom langgraph.prebuilt import create_react_agent\n\n# Get the prompt to use - you can modify this!\nprompt = hub.pull(\"wfh/react-agent-executor\")\nprompt.pretty_print()\n\n# Choose the LLM that will drive the agent\nllm = ChatOpenAI(model=\"gpt-4-turbo-preview\")\nagent_executor = create_react_agent(llm, tools, messages_modifier=prompt)"]
+ "source": [
+ "from langchain import hub\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "from langgraph.prebuilt import create_react_agent\n",
+ "\n",
+ "# Get the prompt to use - you can modify this!\n",
+ "prompt = hub.pull(\"wfh/react-agent-executor\")\n",
+ "prompt.pretty_print()\n",
+ "\n",
+ "# Choose the LLM that will drive the agent\n",
+ "llm = ChatOpenAI(model=\"gpt-4-turbo-preview\")\n",
+ "agent_executor = create_react_agent(llm, tools, messages_modifier=prompt)"
+ ]
},
{
"cell_type": "code",
@@ -150,7 +186,9 @@
"output_type": "execute_result"
}
],
- "source": ["agent_executor.invoke({\"messages\": [(\"user\", \"who is the winnner of the us open\")]})"]
+ "source": [
+ "agent_executor.invoke({\"messages\": [(\"user\", \"who is the winnner of the us open\")]})"
+ ]
},
{
"cell_type": "markdown",
@@ -174,7 +212,17 @@
"id": "8eeeaeea-8f10-4fbe-8e24-4e1a2381a009",
"metadata": {},
"outputs": [],
- "source": ["import operator\nfrom typing import Annotated, List, Tuple, TypedDict\n\n\nclass PlanExecute(TypedDict):\n input: str\n plan: List[str]\n past_steps: Annotated[List[Tuple], operator.add]\n response: str"]
+ "source": [
+ "import operator\n",
+ "from typing import Annotated, List, Tuple, TypedDict\n",
+ "\n",
+ "\n",
+ "class PlanExecute(TypedDict):\n",
+ " input: str\n",
+ " plan: List[str]\n",
+ " past_steps: Annotated[List[Tuple], operator.add]\n",
+ " response: str"
+ ]
},
{
"cell_type": "markdown",
@@ -192,7 +240,17 @@
"id": "4a88626d-6dfd-4488-87f0-a9a0dd6da44c",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.pydantic_v1 import BaseModel, Field\n\n\nclass Plan(BaseModel):\n \"\"\"Plan to follow in future\"\"\"\n\n steps: List[str] = Field(\n description=\"different steps to follow, should be in sorted order\"\n )"]
+ "source": [
+ "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+ "\n",
+ "\n",
+ "class Plan(BaseModel):\n",
+ " \"\"\"Plan to follow in future\"\"\"\n",
+ "\n",
+ " steps: List[str] = Field(\n",
+ " description=\"different steps to follow, should be in sorted order\"\n",
+ " )"
+ ]
},
{
"cell_type": "code",
@@ -200,7 +258,24 @@
"id": "ec7b1867-1ea3-4df3-9a98-992a1c32ec49",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.prompts import ChatPromptTemplate\n\nplanner_prompt = ChatPromptTemplate.from_messages(\n [\n (\n \"system\",\n \"\"\"For the given objective, come up with a simple step by step plan. \\\nThis plan should involve individual tasks, that if executed correctly will yield the correct answer. Do not add any superfluous steps. \\\nThe result of the final step should be the final answer. Make sure that each step has all the information needed - do not skip steps.\"\"\",\n ),\n (\"placeholder\", \"{messages}\"),\n ]\n)\nplanner = planner_prompt | ChatOpenAI(\n model=\"gpt-4o\", temperature=0\n).with_structured_output(Plan)"]
+ "source": [
+ "from langchain_core.prompts import ChatPromptTemplate\n",
+ "\n",
+ "planner_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\n",
+ " \"system\",\n",
+ " \"\"\"For the given objective, come up with a simple step by step plan. \\\n",
+ "This plan should involve individual tasks, that if executed correctly will yield the correct answer. Do not add any superfluous steps. \\\n",
+ "The result of the final step should be the final answer. Make sure that each step has all the information needed - do not skip steps.\"\"\",\n",
+ " ),\n",
+ " (\"placeholder\", \"{messages}\"),\n",
+ " ]\n",
+ ")\n",
+ "planner = planner_prompt | ChatOpenAI(\n",
+ " model=\"gpt-4o\", temperature=0\n",
+ ").with_structured_output(Plan)"
+ ]
},
{
"cell_type": "code",
@@ -219,7 +294,15 @@
"output_type": "execute_result"
}
],
- "source": ["planner.invoke(\n {\n \"messages\": [\n (\"user\", \"what is the hometown of the current Australia open winner?\")\n ]\n }\n)"]
+ "source": [
+ "planner.invoke(\n",
+ " {\n",
+ " \"messages\": [\n",
+ " (\"user\", \"what is the hometown of the current Australia open winner?\")\n",
+ " ]\n",
+ " }\n",
+ ")"
+ ]
},
{
"cell_type": "markdown",
@@ -237,7 +320,47 @@
"id": "ec2d12cc-016a-44d1-aa08-4c5ce1e8fe2a",
"metadata": {},
"outputs": [],
- "source": ["from typing import Union\n\n\nclass Response(BaseModel):\n \"\"\"Response to user.\"\"\"\n\n response: str\n\n\nclass Act(BaseModel):\n \"\"\"Action to perform.\"\"\"\n\n action: Union[Response, Plan] = Field(\n description=\"Action to perform. If you want to respond to user, use Response. \"\n \"If you need to further use tools to get the answer, use Plan.\"\n )\n\n\nreplanner_prompt = ChatPromptTemplate.from_template(\n \"\"\"For the given objective, come up with a simple step by step plan. \\\nThis plan should involve individual tasks, that if executed correctly will yield the correct answer. Do not add any superfluous steps. \\\nThe result of the final step should be the final answer. Make sure that each step has all the information needed - do not skip steps.\n\nYour objective was this:\n{input}\n\nYour original plan was this:\n{plan}\n\nYou have currently done the follow steps:\n{past_steps}\n\nUpdate your plan accordingly. If no more steps are needed and you can return to the user, then respond with that. Otherwise, fill out the plan. Only add steps to the plan that still NEED to be done. Do not return previously done steps as part of the plan.\"\"\"\n)\n\n\nreplanner = replanner_prompt | ChatOpenAI(\n model=\"gpt-4o\", temperature=0\n).with_structured_output(Act)"]
+ "source": [
+ "from typing import Union\n",
+ "\n",
+ "\n",
+ "class Response(BaseModel):\n",
+ " \"\"\"Response to user.\"\"\"\n",
+ "\n",
+ " response: str\n",
+ "\n",
+ "\n",
+ "class Act(BaseModel):\n",
+ " \"\"\"Action to perform.\"\"\"\n",
+ "\n",
+ " action: Union[Response, Plan] = Field(\n",
+ " description=\"Action to perform. If you want to respond to user, use Response. \"\n",
+ " \"If you need to further use tools to get the answer, use Plan.\"\n",
+ " )\n",
+ "\n",
+ "\n",
+ "replanner_prompt = ChatPromptTemplate.from_template(\n",
+ " \"\"\"For the given objective, come up with a simple step by step plan. \\\n",
+ "This plan should involve individual tasks, that if executed correctly will yield the correct answer. Do not add any superfluous steps. \\\n",
+ "The result of the final step should be the final answer. Make sure that each step has all the information needed - do not skip steps.\n",
+ "\n",
+ "Your objective was this:\n",
+ "{input}\n",
+ "\n",
+ "Your original plan was this:\n",
+ "{plan}\n",
+ "\n",
+ "You have currently done the follow steps:\n",
+ "{past_steps}\n",
+ "\n",
+ "Update your plan accordingly. If no more steps are needed and you can return to the user, then respond with that. Otherwise, fill out the plan. Only add steps to the plan that still NEED to be done. Do not return previously done steps as part of the plan.\"\"\"\n",
+ ")\n",
+ "\n",
+ "\n",
+ "replanner = replanner_prompt | ChatOpenAI(\n",
+ " model=\"gpt-4o\", temperature=0\n",
+ ").with_structured_output(Act)"
+ ]
},
{
"cell_type": "markdown",
@@ -255,7 +378,43 @@
"id": "6c8e0dad-bcea-4c9a-8922-0d820892e2d0",
"metadata": {},
"outputs": [],
- "source": ["from typing import Literal\n\n\nasync def execute_step(state: PlanExecute):\n plan = state[\"plan\"]\n plan_str = \"\\n\".join(f\"{i+1}. {step}\" for i, step in enumerate(plan))\n task = plan[0]\n task_formatted = f\"\"\"For the following plan:\n{plan_str}\\n\\nYou are tasked with executing step {1}, {task}.\"\"\"\n agent_response = await agent_executor.ainvoke(\n {\"messages\": [(\"user\", task_formatted)]}\n )\n return {\n \"past_steps\": (task, agent_response[\"messages\"][-1].content),\n }\n\n\nasync def plan_step(state: PlanExecute):\n plan = await planner.ainvoke({\"messages\": [(\"user\", state[\"input\"])]})\n return {\"plan\": plan.steps}\n\n\nasync def replan_step(state: PlanExecute):\n output = await replanner.ainvoke(state)\n if isinstance(output.action, Response):\n return {\"response\": output.action.response}\n else:\n return {\"plan\": output.action.steps}\n\n\ndef should_end(state: PlanExecute) -> Literal[\"agent\", \"__end__\"]:\n if \"response\" in state and state[\"response\"]:\n return \"__end__\"\n else:\n return \"agent\""]
+ "source": [
+ "from typing import Literal\n",
+ "\n",
+ "\n",
+ "async def execute_step(state: PlanExecute):\n",
+ " plan = state[\"plan\"]\n",
+ " plan_str = \"\\n\".join(f\"{i+1}. {step}\" for i, step in enumerate(plan))\n",
+ " task = plan[0]\n",
+ " task_formatted = f\"\"\"For the following plan:\n",
+ "{plan_str}\\n\\nYou are tasked with executing step {1}, {task}.\"\"\"\n",
+ " agent_response = await agent_executor.ainvoke(\n",
+ " {\"messages\": [(\"user\", task_formatted)]}\n",
+ " )\n",
+ " return {\n",
+ " \"past_steps\": (task, agent_response[\"messages\"][-1].content),\n",
+ " }\n",
+ "\n",
+ "\n",
+ "async def plan_step(state: PlanExecute):\n",
+ " plan = await planner.ainvoke({\"messages\": [(\"user\", state[\"input\"])]})\n",
+ " return {\"plan\": plan.steps}\n",
+ "\n",
+ "\n",
+ "async def replan_step(state: PlanExecute):\n",
+ " output = await replanner.ainvoke(state)\n",
+ " if isinstance(output.action, Response):\n",
+ " return {\"response\": output.action.response}\n",
+ " else:\n",
+ " return {\"plan\": output.action.steps}\n",
+ "\n",
+ "\n",
+ "def should_end(state: PlanExecute) -> Literal[\"agent\", \"__end__\"]:\n",
+ " if \"response\" in state and state[\"response\"]:\n",
+ " return \"__end__\"\n",
+ " else:\n",
+ " return \"agent\""
+ ]
},
{
"cell_type": "code",
@@ -263,7 +422,39 @@
"id": "e954cea0-5ccc-46c2-a27b-f5b7185b597d",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.graph import StateGraph, START\n\nworkflow = StateGraph(PlanExecute)\n\n# Add the plan node\nworkflow.add_node(\"planner\", plan_step)\n\n# Add the execution step\nworkflow.add_node(\"agent\", execute_step)\n\n# Add a replan node\nworkflow.add_node(\"replan\", replan_step)\n\nworkflow.add_edge(START, \"planner\")\n\n# From plan we go to agent\nworkflow.add_edge(\"planner\", \"agent\")\n\n# From agent, we replan\nworkflow.add_edge(\"agent\", \"replan\")\n\nworkflow.add_conditional_edges(\n \"replan\",\n # Next, we pass in the function that will determine which node is called next.\n should_end,\n)\n\n# Finally, we compile it!\n# This compiles it into a LangChain Runnable,\n# meaning you can use it as you would any other runnable\napp = workflow.compile()"]
+ "source": [
+ "from langgraph.graph import StateGraph, START\n",
+ "\n",
+ "workflow = StateGraph(PlanExecute)\n",
+ "\n",
+ "# Add the plan node\n",
+ "workflow.add_node(\"planner\", plan_step)\n",
+ "\n",
+ "# Add the execution step\n",
+ "workflow.add_node(\"agent\", execute_step)\n",
+ "\n",
+ "# Add a replan node\n",
+ "workflow.add_node(\"replan\", replan_step)\n",
+ "\n",
+ "workflow.add_edge(START, \"planner\")\n",
+ "\n",
+ "# From plan we go to agent\n",
+ "workflow.add_edge(\"planner\", \"agent\")\n",
+ "\n",
+ "# From agent, we replan\n",
+ "workflow.add_edge(\"agent\", \"replan\")\n",
+ "\n",
+ "workflow.add_conditional_edges(\n",
+ " \"replan\",\n",
+ " # Next, we pass in the function that will determine which node is called next.\n",
+ " should_end,\n",
+ ")\n",
+ "\n",
+ "# Finally, we compile it!\n",
+ "# This compiles it into a LangChain Runnable,\n",
+ "# meaning you can use it as you would any other runnable\n",
+ "app = workflow.compile()"
+ ]
},
{
"cell_type": "code",
@@ -282,7 +473,11 @@
"output_type": "display_data"
}
],
- "source": ["from IPython.display import Image, display\n\ndisplay(Image(app.get_graph(xray=True).draw_mermaid_png()))"]
+ "source": [
+ "from IPython.display import Image, display\n",
+ "\n",
+ "display(Image(app.get_graph(xray=True).draw_mermaid_png()))"
+ ]
},
{
"cell_type": "code",
@@ -310,7 +505,14 @@
]
}
],
- "source": ["config = {\"recursion_limit\": 50}\ninputs = {\"input\": \"what is the hometown of the 2024 Australia open winner?\"}\nasync for event in app.astream(inputs, config=config):\n for k, v in event.items():\n if k != \"__end__\":\n print(v)"]
+ "source": [
+ "config = {\"recursion_limit\": 50}\n",
+ "inputs = {\"input\": \"what is the hometown of the 2024 Australia open winner?\"}\n",
+ "async for event in app.astream(inputs, config=config):\n",
+ " for k, v in event.items():\n",
+ " if k != \"__end__\":\n",
+ " print(v)"
+ ]
},
{
"cell_type": "markdown",
@@ -321,14 +523,6 @@
"\n",
"Congrats on making a plan-and-execute agent! One known limitations of the above design is that each task is still executed in sequence, meaning embarrassingly parallel operations all add to the total execution time. You could improve on this by having each task represented as a DAG (similar to LLMCompiler), rather than a regular list."
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ad8f7955-2cc9-4ebb-8c41-13abb3351a24",
- "metadata": {},
- "outputs": [],
- "source": [""]
}
],
"metadata": {
diff --git a/examples/rag/langgraph_adaptive_rag.ipynb b/examples/rag/langgraph_adaptive_rag.ipynb
index 8caa57bfa..f612183c5 100644
--- a/examples/rag/langgraph_adaptive_rag.ipynb
+++ b/examples/rag/langgraph_adaptive_rag.ipynb
@@ -47,7 +47,8 @@
"metadata": {},
"outputs": [],
"source": [
- "%%capture --no-stderr\n! pip install -U langchain_community tiktoken langchain-openai langchain-cohere langchainhub chromadb langchain langgraph tavily-python"
+ "%%capture --no-stderr\n",
+ "! pip install -U langchain_community tiktoken langchain-openai langchain-cohere langchainhub chromadb langchain langgraph tavily-python"
]
},
{
@@ -57,7 +58,12 @@
"metadata": {},
"outputs": [],
"source": [
- "### LLMs\nimport os\n\nos.environ[\"OPENAI_API_KEY\"] = \"\"\nos.environ[\"COHERE_API_KEY\"] = \"\"\nos.environ[\"TAVILY_API_KEY\"] = \"\""
+ "### LLMs\n",
+ "import os\n",
+ "\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
+ "os.environ[\"COHERE_API_KEY\"] = \"\"\n",
+ "os.environ[\"TAVILY_API_KEY\"] = \"\""
]
},
{
@@ -77,7 +83,10 @@
"metadata": {},
"outputs": [],
"source": [
- "### Tracing (optional)\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"\""
+ "### Tracing (optional)\n",
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+ "os.environ[\"LANGCHAIN_API_KEY\"] = \"\""
]
},
{
@@ -95,7 +104,42 @@
"metadata": {},
"outputs": [],
"source": [
- "### Build Index\n\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_openai import OpenAIEmbeddings\n\n### from langchain_cohere import CohereEmbeddings\n\n# Set embeddings\nembd = OpenAIEmbeddings()\n\n# Docs to index\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\n# Load\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\n# Split\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=500, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorstore\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=embd,\n)\nretriever = vectorstore.as_retriever()"
+ "### Build Index\n",
+ "\n",
+ "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+ "from langchain_community.document_loaders import WebBaseLoader\n",
+ "from langchain_community.vectorstores import Chroma\n",
+ "from langchain_openai import OpenAIEmbeddings\n",
+ "\n",
+ "### from langchain_cohere import CohereEmbeddings\n",
+ "\n",
+ "# Set embeddings\n",
+ "embd = OpenAIEmbeddings()\n",
+ "\n",
+ "# Docs to index\n",
+ "urls = [\n",
+ " \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+ "]\n",
+ "\n",
+ "# Load\n",
+ "docs = [WebBaseLoader(url).load() for url in urls]\n",
+ "docs_list = [item for sublist in docs for item in sublist]\n",
+ "\n",
+ "# Split\n",
+ "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+ " chunk_size=500, chunk_overlap=0\n",
+ ")\n",
+ "doc_splits = text_splitter.split_documents(docs_list)\n",
+ "\n",
+ "# Add to vectorstore\n",
+ "vectorstore = Chroma.from_documents(\n",
+ " documents=doc_splits,\n",
+ " collection_name=\"rag-chroma\",\n",
+ " embedding=embd,\n",
+ ")\n",
+ "retriever = vectorstore.as_retriever()"
]
},
{
@@ -122,7 +166,47 @@
}
],
"source": [
- "### Router\n\nfrom typing import Literal\n\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\n\n# Data model\nclass RouteQuery(BaseModel):\n \"\"\"Route a user query to the most relevant datasource.\"\"\"\n\n datasource: Literal[\"vectorstore\", \"web_search\"] = Field(\n ...,\n description=\"Given a user question choose to route it to web search or a vectorstore.\",\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_router = llm.with_structured_output(RouteQuery)\n\n# Prompt\nsystem = \"\"\"You are an expert at routing a user question to a vectorstore or web search.\nThe vectorstore contains documents related to agents, prompt engineering, and adversarial attacks.\nUse the vectorstore for questions on these topics. Otherwise, use web-search.\"\"\"\nroute_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"{question}\"),\n ]\n)\n\nquestion_router = route_prompt | structured_llm_router\nprint(\n question_router.invoke(\n {\"question\": \"Who will the Bears draft first in the NFL draft?\"}\n )\n)\nprint(question_router.invoke({\"question\": \"What are the types of agent memory?\"}))"
+ "### Router\n",
+ "\n",
+ "from typing import Literal\n",
+ "\n",
+ "from langchain_core.prompts import ChatPromptTemplate\n",
+ "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class RouteQuery(BaseModel):\n",
+ " \"\"\"Route a user query to the most relevant datasource.\"\"\"\n",
+ "\n",
+ " datasource: Literal[\"vectorstore\", \"web_search\"] = Field(\n",
+ " ...,\n",
+ " description=\"Given a user question choose to route it to web search or a vectorstore.\",\n",
+ " )\n",
+ "\n",
+ "\n",
+ "# LLM with function call\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "structured_llm_router = llm.with_structured_output(RouteQuery)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You are an expert at routing a user question to a vectorstore or web search.\n",
+ "The vectorstore contains documents related to agents, prompt engineering, and adversarial attacks.\n",
+ "Use the vectorstore for questions on these topics. Otherwise, use web-search.\"\"\"\n",
+ "route_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\"human\", \"{question}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "question_router = route_prompt | structured_llm_router\n",
+ "print(\n",
+ " question_router.invoke(\n",
+ " {\"question\": \"Who will the Bears draft first in the NFL draft?\"}\n",
+ " )\n",
+ ")\n",
+ "print(question_router.invoke({\"question\": \"What are the types of agent memory?\"}))"
]
},
{
@@ -140,7 +224,39 @@
}
],
"source": [
- "### Retrieval Grader\n\n\n# Data model\nclass GradeDocuments(BaseModel):\n \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n\n binary_score: str = Field(\n description=\"Documents are relevant to the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeDocuments)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\ngrade_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n ]\n)\n\nretrieval_grader = grade_prompt | structured_llm_grader\nquestion = \"agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+ "### Retrieval Grader\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class GradeDocuments(BaseModel):\n",
+ " \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
+ "\n",
+ " binary_score: str = Field(\n",
+ " description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
+ " )\n",
+ "\n",
+ "\n",
+ "# LLM with function call\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+ " If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
+ " It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
+ " Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
+ "grade_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "retrieval_grader = grade_prompt | structured_llm_grader\n",
+ "question = \"agent memory\"\n",
+ "docs = retriever.get_relevant_documents(question)\n",
+ "doc_txt = docs[1].page_content\n",
+ "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
]
},
{
@@ -158,7 +274,29 @@
}
],
"source": [
- "### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n\n\n# Post-processing\ndef format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"
+ "### Generate\n",
+ "\n",
+ "from langchain import hub\n",
+ "from langchain_core.output_parsers import StrOutputParser\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
+ "\n",
+ "\n",
+ "# Post-processing\n",
+ "def format_docs(docs):\n",
+ " return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+ "\n",
+ "\n",
+ "# Chain\n",
+ "rag_chain = prompt | llm | StrOutputParser()\n",
+ "\n",
+ "# Run\n",
+ "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+ "print(generation)"
]
},
{
@@ -179,7 +317,34 @@
}
],
"source": [
- "### Hallucination Grader\n\n\n# Data model\nclass GradeHallucinations(BaseModel):\n \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n\n binary_score: str = Field(\n description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeHallucinations)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n \n Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\nhallucination_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n ]\n)\n\nhallucination_grader = hallucination_prompt | structured_llm_grader\nhallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+ "### Hallucination Grader\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class GradeHallucinations(BaseModel):\n",
+ " \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n",
+ "\n",
+ " binary_score: str = Field(\n",
+ " description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n",
+ " )\n",
+ "\n",
+ "\n",
+ "# LLM with function call\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "structured_llm_grader = llm.with_structured_output(GradeHallucinations)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n \n",
+ " Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\n",
+ "hallucination_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "hallucination_grader = hallucination_prompt | structured_llm_grader\n",
+ "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
]
},
{
@@ -200,7 +365,34 @@
}
],
"source": [
- "### Answer Grader\n\n\n# Data model\nclass GradeAnswer(BaseModel):\n \"\"\"Binary score to assess answer addresses question.\"\"\"\n\n binary_score: str = Field(\n description=\"Answer addresses the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeAnswer)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n \n Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\nanswer_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n ]\n)\n\nanswer_grader = answer_prompt | structured_llm_grader\nanswer_grader.invoke({\"question\": question, \"generation\": generation})"
+ "### Answer Grader\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class GradeAnswer(BaseModel):\n",
+ " \"\"\"Binary score to assess answer addresses question.\"\"\"\n",
+ "\n",
+ " binary_score: str = Field(\n",
+ " description=\"Answer addresses the question, 'yes' or 'no'\"\n",
+ " )\n",
+ "\n",
+ "\n",
+ "# LLM with function call\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "structured_llm_grader = llm.with_structured_output(GradeAnswer)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n \n",
+ " Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\n",
+ "answer_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "answer_grader = answer_prompt | structured_llm_grader\n",
+ "answer_grader.invoke({\"question\": question, \"generation\": generation})"
]
},
{
@@ -221,7 +413,26 @@
}
],
"source": [
- "### Question Re-writer\n\n# LLM\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n\n# Prompt\nsystem = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\nre_write_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\n \"human\",\n \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n ),\n ]\n)\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nquestion_rewriter.invoke({\"question\": question})"
+ "### Question Re-writer\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+ " for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
+ "re_write_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\n",
+ " \"human\",\n",
+ " \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
+ " ),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+ "question_rewriter.invoke({\"question\": question})"
]
},
{
@@ -239,7 +450,11 @@
"metadata": {},
"outputs": [],
"source": [
- "### Search\n\nfrom langchain_community.tools.tavily_search import TavilySearchResults\n\nweb_search_tool = TavilySearchResults(k=3)"
+ "### Search\n",
+ "\n",
+ "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+ "\n",
+ "web_search_tool = TavilySearchResults(k=3)"
]
},
{
@@ -261,7 +476,24 @@
"metadata": {},
"outputs": [],
"source": [
- "from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n documents: List[str]"
+ "from typing import List\n",
+ "\n",
+ "from typing_extensions import TypedDict\n",
+ "\n",
+ "\n",
+ "class GraphState(TypedDict):\n",
+ " \"\"\"\n",
+ " Represents the state of our graph.\n",
+ "\n",
+ " Attributes:\n",
+ " question: question\n",
+ " generation: LLM generation\n",
+ " documents: list of documents\n",
+ " \"\"\"\n",
+ "\n",
+ " question: str\n",
+ " generation: str\n",
+ " documents: List[str]"
]
},
{
@@ -279,7 +511,209 @@
"metadata": {},
"outputs": [],
"source": [
- "from langchain.schema import Document\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.invoke(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n continue\n return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}\n\n\ndef web_search(state):\n \"\"\"\n Web search based on the re-phrased question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with appended web results\n \"\"\"\n\n print(\"---WEB SEARCH---\")\n question = state[\"question\"]\n\n # Web search\n docs = web_search_tool.invoke({\"query\": question})\n web_results = \"\\n\".join([d[\"content\"] for d in docs])\n web_results = Document(page_content=web_results)\n\n return {\"documents\": web_results, \"question\": question}\n\n\n### Edges ###\n\n\ndef route_question(state):\n \"\"\"\n Route question to web search or RAG.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Next node to call\n \"\"\"\n\n print(\"---ROUTE QUESTION---\")\n question = state[\"question\"]\n source = question_router.invoke({\"question\": question})\n if source.datasource == \"web_search\":\n print(\"---ROUTE QUESTION TO WEB SEARCH---\")\n return \"web_search\"\n elif source.datasource == \"vectorstore\":\n print(\"---ROUTE QUESTION TO RAG---\")\n return \"vectorstore\"\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n filtered_documents = state[\"documents\"]\n\n if not filtered_documents:\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\"\n\n\ndef grade_generation_v_documents_and_question(state):\n \"\"\"\n Determines whether the generation is grounded in the document and answers question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Decision for next node to call\n \"\"\"\n\n print(\"---CHECK HALLUCINATIONS---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = state[\"generation\"]\n\n score = hallucination_grader.invoke(\n {\"documents\": documents, \"generation\": generation}\n )\n grade = score.binary_score\n\n # Check hallucination\n if grade == \"yes\":\n print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n # Check question-answering\n print(\"---GRADE GENERATION vs QUESTION---\")\n score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n return \"useful\"\n else:\n print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n return \"not useful\"\n else:\n pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n return \"not supported\""
+ "from langchain.schema import Document\n",
+ "\n",
+ "\n",
+ "def retrieve(state):\n",
+ " \"\"\"\n",
+ " Retrieve documents\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, documents, that contains retrieved documents\n",
+ " \"\"\"\n",
+ " print(\"---RETRIEVE---\")\n",
+ " question = state[\"question\"]\n",
+ "\n",
+ " # Retrieval\n",
+ " documents = retriever.invoke(question)\n",
+ " return {\"documents\": documents, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def generate(state):\n",
+ " \"\"\"\n",
+ " Generate answer\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, generation, that contains LLM generation\n",
+ " \"\"\"\n",
+ " print(\"---GENERATE---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # RAG generation\n",
+ " generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+ " return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+ "\n",
+ "\n",
+ "def grade_documents(state):\n",
+ " \"\"\"\n",
+ " Determines whether the retrieved documents are relevant to the question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with only filtered relevant documents\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Score each doc\n",
+ " filtered_docs = []\n",
+ " for d in documents:\n",
+ " score = retrieval_grader.invoke(\n",
+ " {\"question\": question, \"document\": d.page_content}\n",
+ " )\n",
+ " grade = score.binary_score\n",
+ " if grade == \"yes\":\n",
+ " print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+ " filtered_docs.append(d)\n",
+ " else:\n",
+ " print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+ " continue\n",
+ " return {\"documents\": filtered_docs, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def transform_query(state):\n",
+ " \"\"\"\n",
+ " Transform the query to produce a better question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates question key with a re-phrased question\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---TRANSFORM QUERY---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Re-write question\n",
+ " better_question = question_rewriter.invoke({\"question\": question})\n",
+ " return {\"documents\": documents, \"question\": better_question}\n",
+ "\n",
+ "\n",
+ "def web_search(state):\n",
+ " \"\"\"\n",
+ " Web search based on the re-phrased question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with appended web results\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---WEB SEARCH---\")\n",
+ " question = state[\"question\"]\n",
+ "\n",
+ " # Web search\n",
+ " docs = web_search_tool.invoke({\"query\": question})\n",
+ " web_results = \"\\n\".join([d[\"content\"] for d in docs])\n",
+ " web_results = Document(page_content=web_results)\n",
+ "\n",
+ " return {\"documents\": web_results, \"question\": question}\n",
+ "\n",
+ "\n",
+ "### Edges ###\n",
+ "\n",
+ "\n",
+ "def route_question(state):\n",
+ " \"\"\"\n",
+ " Route question to web search or RAG.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---ROUTE QUESTION---\")\n",
+ " question = state[\"question\"]\n",
+ " source = question_router.invoke({\"question\": question})\n",
+ " if source.datasource == \"web_search\":\n",
+ " print(\"---ROUTE QUESTION TO WEB SEARCH---\")\n",
+ " return \"web_search\"\n",
+ " elif source.datasource == \"vectorstore\":\n",
+ " print(\"---ROUTE QUESTION TO RAG---\")\n",
+ " return \"vectorstore\"\n",
+ "\n",
+ "\n",
+ "def decide_to_generate(state):\n",
+ " \"\"\"\n",
+ " Determines whether to generate an answer, or re-generate a question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Binary decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+ " state[\"question\"]\n",
+ " filtered_documents = state[\"documents\"]\n",
+ "\n",
+ " if not filtered_documents:\n",
+ " # All documents have been filtered check_relevance\n",
+ " # We will re-generate a new query\n",
+ " print(\n",
+ " \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+ " )\n",
+ " return \"transform_query\"\n",
+ " else:\n",
+ " # We have relevant documents, so generate answer\n",
+ " print(\"---DECISION: GENERATE---\")\n",
+ " return \"generate\"\n",
+ "\n",
+ "\n",
+ "def grade_generation_v_documents_and_question(state):\n",
+ " \"\"\"\n",
+ " Determines whether the generation is grounded in the document and answers question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK HALLUCINATIONS---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ " generation = state[\"generation\"]\n",
+ "\n",
+ " score = hallucination_grader.invoke(\n",
+ " {\"documents\": documents, \"generation\": generation}\n",
+ " )\n",
+ " grade = score.binary_score\n",
+ "\n",
+ " # Check hallucination\n",
+ " if grade == \"yes\":\n",
+ " print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
+ " # Check question-answering\n",
+ " print(\"---GRADE GENERATION vs QUESTION---\")\n",
+ " score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
+ " grade = score.binary_score\n",
+ " if grade == \"yes\":\n",
+ " print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
+ " return \"useful\"\n",
+ " else:\n",
+ " print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
+ " return \"not useful\"\n",
+ " else:\n",
+ " pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
+ " return \"not supported\""
]
},
{
@@ -373,7 +807,22 @@
}
],
"source": [
- "from pprint import pprint\n\n# Run\ninputs = {\n \"question\": \"What player at the Bears expected to draft first in the 2024 NFL draft?\"\n}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"
+ "from pprint import pprint\n",
+ "\n",
+ "# Run\n",
+ "inputs = {\n",
+ " \"question\": \"What player at the Bears expected to draft first in the 2024 NFL draft?\"\n",
+ "}\n",
+ "for output in app.stream(inputs):\n",
+ " for key, value in output.items():\n",
+ " # Node\n",
+ " pprint(f\"Node '{key}':\")\n",
+ " # Optional: print full state at each node\n",
+ " # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+ " pprint(\"\\n---\\n\")\n",
+ "\n",
+ "# Final generation\n",
+ "pprint(value[\"generation\"])"
]
},
{
@@ -426,7 +875,18 @@
}
],
"source": [
- "# Run\ninputs = {\"question\": \"What are the types of agent memory?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"
+ "# Run\n",
+ "inputs = {\"question\": \"What are the types of agent memory?\"}\n",
+ "for output in app.stream(inputs):\n",
+ " for key, value in output.items():\n",
+ " # Node\n",
+ " pprint(f\"Node '{key}':\")\n",
+ " # Optional: print full state at each node\n",
+ " # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+ " pprint(\"\\n---\\n\")\n",
+ "\n",
+ "# Final generation\n",
+ "pprint(value[\"generation\"])"
]
},
{
@@ -438,16 +898,6 @@
"\n",
"https://smith.langchain.com/public/fdf0a180-6d15-4d09-bb92-f84f2105ca51/r"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "19ac1f6f-2d84-488f-8a0e-7ee2a46b0f71",
- "metadata": {},
- "outputs": [],
- "source": [
- ""
- ]
}
],
"metadata": {
diff --git a/examples/rag/langgraph_adaptive_rag_local.ipynb b/examples/rag/langgraph_adaptive_rag_local.ipynb
index 500efc649..f666536db 100644
--- a/examples/rag/langgraph_adaptive_rag_local.ipynb
+++ b/examples/rag/langgraph_adaptive_rag_local.ipynb
@@ -45,7 +45,8 @@
"metadata": {},
"outputs": [],
"source": [
- "%capture --no-stderr\n%pip install -U langchain-nomic langchain_community tiktoken langchainhub chromadb langchain langgraph tavily-python nomic[local]"
+ "%capture --no-stderr\n",
+ "%pip install -U langchain-nomic langchain_community tiktoken langchainhub chromadb langchain langgraph tavily-python nomic[local]"
]
},
{
@@ -79,7 +80,8 @@
"metadata": {},
"outputs": [],
"source": [
- "# Ollama model name\nlocal_llm = \"mistral\""
+ "# Ollama model name\n",
+ "local_llm = \"mistral\""
]
},
{
@@ -99,7 +101,11 @@
"metadata": {},
"outputs": [],
"source": [
- "import os\n\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"\""
+ "import os\n",
+ "\n",
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+ "os.environ[\"LANGCHAIN_API_KEY\"] = \"\""
]
},
{
@@ -117,7 +123,32 @@
"metadata": {},
"outputs": [],
"source": [
- "from langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_nomic.embeddings import NomicEmbeddings\n\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=250, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorDB\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n)\nretriever = vectorstore.as_retriever()"
+ "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+ "from langchain_community.document_loaders import WebBaseLoader\n",
+ "from langchain_community.vectorstores import Chroma\n",
+ "from langchain_nomic.embeddings import NomicEmbeddings\n",
+ "\n",
+ "urls = [\n",
+ " \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+ "]\n",
+ "\n",
+ "docs = [WebBaseLoader(url).load() for url in urls]\n",
+ "docs_list = [item for sublist in docs for item in sublist]\n",
+ "\n",
+ "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+ " chunk_size=250, chunk_overlap=0\n",
+ ")\n",
+ "doc_splits = text_splitter.split_documents(docs_list)\n",
+ "\n",
+ "# Add to vectorDB\n",
+ "vectorstore = Chroma.from_documents(\n",
+ " documents=doc_splits,\n",
+ " collection_name=\"rag-chroma\",\n",
+ " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n",
+ ")\n",
+ "retriever = vectorstore.as_retriever()"
]
},
{
@@ -145,7 +176,30 @@
}
],
"source": [
- "### Router\n\nfrom langchain.prompts import PromptTemplate\nfrom langchain_community.chat_models import ChatOllama\nfrom langchain_core.output_parsers import JsonOutputParser\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\nprompt = PromptTemplate(\n template=\"\"\"You are an expert at routing a user question to a vectorstore or web search. \\n\n Use the vectorstore for questions on LLM agents, prompt engineering, and adversarial attacks. \\n\n You do not need to be stringent with the keywords in the question related to these topics. \\n\n Otherwise, use web-search. Give a binary choice 'web_search' or 'vectorstore' based on the question. \\n\n Return the a JSON with a single key 'datasource' and no premable or explanation. \\n\n Question to route: {question}\"\"\",\n input_variables=[\"question\"],\n)\n\nquestion_router = prompt | llm | JsonOutputParser()\nquestion = \"llm agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(question_router.invoke({\"question\": question}))"
+ "### Router\n",
+ "\n",
+ "from langchain.prompts import PromptTemplate\n",
+ "from langchain_community.chat_models import ChatOllama\n",
+ "from langchain_core.output_parsers import JsonOutputParser\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+ "\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are an expert at routing a user question to a vectorstore or web search. \\n\n",
+ " Use the vectorstore for questions on LLM agents, prompt engineering, and adversarial attacks. \\n\n",
+ " You do not need to be stringent with the keywords in the question related to these topics. \\n\n",
+ " Otherwise, use web-search. Give a binary choice 'web_search' or 'vectorstore' based on the question. \\n\n",
+ " Return the a JSON with a single key 'datasource' and no premable or explanation. \\n\n",
+ " Question to route: {question}\"\"\",\n",
+ " input_variables=[\"question\"],\n",
+ ")\n",
+ "\n",
+ "question_router = prompt | llm | JsonOutputParser()\n",
+ "question = \"llm agent memory\"\n",
+ "docs = retriever.get_relevant_documents(question)\n",
+ "doc_txt = docs[1].page_content\n",
+ "print(question_router.invoke({\"question\": question}))"
]
},
{
@@ -163,7 +217,31 @@
}
],
"source": [
- "### Retrieval Grader\n\nfrom langchain.prompts import PromptTemplate\nfrom langchain_community.chat_models import ChatOllama\nfrom langchain_core.output_parsers import JsonOutputParser\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n Here is the retrieved document: \\n\\n {document} \\n\\n\n Here is the user question: {question} \\n\n If the document contains keywords related to the user question, grade it as relevant. \\n\n It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\"\"\",\n input_variables=[\"question\", \"document\"],\n)\n\nretrieval_grader = prompt | llm | JsonOutputParser()\nquestion = \"agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+ "### Retrieval Grader\n",
+ "\n",
+ "from langchain.prompts import PromptTemplate\n",
+ "from langchain_community.chat_models import ChatOllama\n",
+ "from langchain_core.output_parsers import JsonOutputParser\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+ "\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+ " Here is the retrieved document: \\n\\n {document} \\n\\n\n",
+ " Here is the user question: {question} \\n\n",
+ " If the document contains keywords related to the user question, grade it as relevant. \\n\n",
+ " It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
+ " Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n",
+ " Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\"\"\",\n",
+ " input_variables=[\"question\", \"document\"],\n",
+ ")\n",
+ "\n",
+ "retrieval_grader = prompt | llm | JsonOutputParser()\n",
+ "question = \"agent memory\"\n",
+ "docs = retriever.get_relevant_documents(question)\n",
+ "doc_txt = docs[1].page_content\n",
+ "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
]
},
{
@@ -181,7 +259,31 @@
}
],
"source": [
- "### Generate\n\nfrom langchain import hub\nfrom langchain_community.chat_models import ChatOllama\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOllama(model=local_llm, temperature=0)\n\n\n# Post-processing\ndef format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\nquestion = \"agent memory\"\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"
+ "### Generate\n",
+ "\n",
+ "from langchain import hub\n",
+ "from langchain_community.chat_models import ChatOllama\n",
+ "from langchain_core.output_parsers import StrOutputParser\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, temperature=0)\n",
+ "\n",
+ "\n",
+ "# Post-processing\n",
+ "def format_docs(docs):\n",
+ " return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+ "\n",
+ "\n",
+ "# Chain\n",
+ "rag_chain = prompt | llm | StrOutputParser()\n",
+ "\n",
+ "# Run\n",
+ "question = \"agent memory\"\n",
+ "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+ "print(generation)"
]
},
{
@@ -202,7 +304,26 @@
}
],
"source": [
- "### Hallucination Grader\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing whether an answer is grounded in / supported by a set of facts. \\n \n Here are the facts:\n \\n ------- \\n\n {documents} \n \\n ------- \\n\n Here is the answer: {generation}\n Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \\n\n Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n input_variables=[\"generation\", \"documents\"],\n)\n\nhallucination_grader = prompt | llm | JsonOutputParser()\nhallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+ "### Hallucination Grader\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are a grader assessing whether an answer is grounded in / supported by a set of facts. \\n \n",
+ " Here are the facts:\n",
+ " \\n ------- \\n\n",
+ " {documents} \n",
+ " \\n ------- \\n\n",
+ " Here is the answer: {generation}\n",
+ " Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \\n\n",
+ " Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
+ " input_variables=[\"generation\", \"documents\"],\n",
+ ")\n",
+ "\n",
+ "hallucination_grader = prompt | llm | JsonOutputParser()\n",
+ "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
]
},
{
@@ -223,7 +344,26 @@
}
],
"source": [
- "### Answer Grader\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing whether an answer is useful to resolve a question. \\n \n Here is the answer:\n \\n ------- \\n\n {generation} \n \\n ------- \\n\n Here is the question: {question}\n Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \\n\n Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n input_variables=[\"generation\", \"question\"],\n)\n\nanswer_grader = prompt | llm | JsonOutputParser()\nanswer_grader.invoke({\"question\": question, \"generation\": generation})"
+ "### Answer Grader\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are a grader assessing whether an answer is useful to resolve a question. \\n \n",
+ " Here is the answer:\n",
+ " \\n ------- \\n\n",
+ " {generation} \n",
+ " \\n ------- \\n\n",
+ " Here is the question: {question}\n",
+ " Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \\n\n",
+ " Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
+ " input_variables=[\"generation\", \"question\"],\n",
+ ")\n",
+ "\n",
+ "answer_grader = prompt | llm | JsonOutputParser()\n",
+ "answer_grader.invoke({\"question\": question, \"generation\": generation})"
]
},
{
@@ -244,7 +384,21 @@
}
],
"source": [
- "### Question Re-writer\n\n# LLM\nllm = ChatOllama(model=local_llm, temperature=0)\n\n# Prompt\nre_write_prompt = PromptTemplate(\n template=\"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n for vectorstore retrieval. Look at the initial and formulate an improved question. \\n\n Here is the initial question: \\n\\n {question}. Improved question with no preamble: \\n \"\"\",\n input_variables=[\"generation\", \"question\"],\n)\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nquestion_rewriter.invoke({\"question\": question})"
+ "### Question Re-writer\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "re_write_prompt = PromptTemplate(\n",
+ " template=\"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+ " for vectorstore retrieval. Look at the initial and formulate an improved question. \\n\n",
+ " Here is the initial question: \\n\\n {question}. Improved question with no preamble: \\n \"\"\",\n",
+ " input_variables=[\"generation\", \"question\"],\n",
+ ")\n",
+ "\n",
+ "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+ "question_rewriter.invoke({\"question\": question})"
]
},
{
@@ -262,7 +416,11 @@
"metadata": {},
"outputs": [],
"source": [
- "### Search\n\nfrom langchain_community.tools.tavily_search import TavilySearchResults\n\nweb_search_tool = TavilySearchResults(k=3)"
+ "### Search\n",
+ "\n",
+ "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+ "\n",
+ "web_search_tool = TavilySearchResults(k=3)"
]
},
{
@@ -284,7 +442,24 @@
"metadata": {},
"outputs": [],
"source": [
- "from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n documents: List[str]"
+ "from typing import List\n",
+ "\n",
+ "from typing_extensions import TypedDict\n",
+ "\n",
+ "\n",
+ "class GraphState(TypedDict):\n",
+ " \"\"\"\n",
+ " Represents the state of our graph.\n",
+ "\n",
+ " Attributes:\n",
+ " question: question\n",
+ " generation: LLM generation\n",
+ " documents: list of documents\n",
+ " \"\"\"\n",
+ "\n",
+ " question: str\n",
+ " generation: str\n",
+ " documents: List[str]"
]
},
{
@@ -294,7 +469,214 @@
"metadata": {},
"outputs": [],
"source": [
- "### Nodes\n\nfrom langchain.schema import Document\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.get_relevant_documents(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score[\"score\"]\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n continue\n return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}\n\n\ndef web_search(state):\n \"\"\"\n Web search based on the re-phrased question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with appended web results\n \"\"\"\n\n print(\"---WEB SEARCH---\")\n question = state[\"question\"]\n\n # Web search\n docs = web_search_tool.invoke({\"query\": question})\n web_results = \"\\n\".join([d[\"content\"] for d in docs])\n web_results = Document(page_content=web_results)\n\n return {\"documents\": web_results, \"question\": question}\n\n\n### Edges ###\n\n\ndef route_question(state):\n \"\"\"\n Route question to web search or RAG.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Next node to call\n \"\"\"\n\n print(\"---ROUTE QUESTION---\")\n question = state[\"question\"]\n print(question)\n source = question_router.invoke({\"question\": question})\n print(source)\n print(source[\"datasource\"])\n if source[\"datasource\"] == \"web_search\":\n print(\"---ROUTE QUESTION TO WEB SEARCH---\")\n return \"web_search\"\n elif source[\"datasource\"] == \"vectorstore\":\n print(\"---ROUTE QUESTION TO RAG---\")\n return \"vectorstore\"\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n filtered_documents = state[\"documents\"]\n\n if not filtered_documents:\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\"\n\n\ndef grade_generation_v_documents_and_question(state):\n \"\"\"\n Determines whether the generation is grounded in the document and answers question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Decision for next node to call\n \"\"\"\n\n print(\"---CHECK HALLUCINATIONS---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = state[\"generation\"]\n\n score = hallucination_grader.invoke(\n {\"documents\": documents, \"generation\": generation}\n )\n grade = score[\"score\"]\n\n # Check hallucination\n if grade == \"yes\":\n print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n # Check question-answering\n print(\"---GRADE GENERATION vs QUESTION---\")\n score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n grade = score[\"score\"]\n if grade == \"yes\":\n print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n return \"useful\"\n else:\n print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n return \"not useful\"\n else:\n pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n return \"not supported\""
+ "### Nodes\n",
+ "\n",
+ "from langchain.schema import Document\n",
+ "\n",
+ "\n",
+ "def retrieve(state):\n",
+ " \"\"\"\n",
+ " Retrieve documents\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, documents, that contains retrieved documents\n",
+ " \"\"\"\n",
+ " print(\"---RETRIEVE---\")\n",
+ " question = state[\"question\"]\n",
+ "\n",
+ " # Retrieval\n",
+ " documents = retriever.get_relevant_documents(question)\n",
+ " return {\"documents\": documents, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def generate(state):\n",
+ " \"\"\"\n",
+ " Generate answer\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, generation, that contains LLM generation\n",
+ " \"\"\"\n",
+ " print(\"---GENERATE---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # RAG generation\n",
+ " generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+ " return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+ "\n",
+ "\n",
+ "def grade_documents(state):\n",
+ " \"\"\"\n",
+ " Determines whether the retrieved documents are relevant to the question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with only filtered relevant documents\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Score each doc\n",
+ " filtered_docs = []\n",
+ " for d in documents:\n",
+ " score = retrieval_grader.invoke(\n",
+ " {\"question\": question, \"document\": d.page_content}\n",
+ " )\n",
+ " grade = score[\"score\"]\n",
+ " if grade == \"yes\":\n",
+ " print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+ " filtered_docs.append(d)\n",
+ " else:\n",
+ " print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+ " continue\n",
+ " return {\"documents\": filtered_docs, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def transform_query(state):\n",
+ " \"\"\"\n",
+ " Transform the query to produce a better question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates question key with a re-phrased question\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---TRANSFORM QUERY---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Re-write question\n",
+ " better_question = question_rewriter.invoke({\"question\": question})\n",
+ " return {\"documents\": documents, \"question\": better_question}\n",
+ "\n",
+ "\n",
+ "def web_search(state):\n",
+ " \"\"\"\n",
+ " Web search based on the re-phrased question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with appended web results\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---WEB SEARCH---\")\n",
+ " question = state[\"question\"]\n",
+ "\n",
+ " # Web search\n",
+ " docs = web_search_tool.invoke({\"query\": question})\n",
+ " web_results = \"\\n\".join([d[\"content\"] for d in docs])\n",
+ " web_results = Document(page_content=web_results)\n",
+ "\n",
+ " return {\"documents\": web_results, \"question\": question}\n",
+ "\n",
+ "\n",
+ "### Edges ###\n",
+ "\n",
+ "\n",
+ "def route_question(state):\n",
+ " \"\"\"\n",
+ " Route question to web search or RAG.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---ROUTE QUESTION---\")\n",
+ " question = state[\"question\"]\n",
+ " print(question)\n",
+ " source = question_router.invoke({\"question\": question})\n",
+ " print(source)\n",
+ " print(source[\"datasource\"])\n",
+ " if source[\"datasource\"] == \"web_search\":\n",
+ " print(\"---ROUTE QUESTION TO WEB SEARCH---\")\n",
+ " return \"web_search\"\n",
+ " elif source[\"datasource\"] == \"vectorstore\":\n",
+ " print(\"---ROUTE QUESTION TO RAG---\")\n",
+ " return \"vectorstore\"\n",
+ "\n",
+ "\n",
+ "def decide_to_generate(state):\n",
+ " \"\"\"\n",
+ " Determines whether to generate an answer, or re-generate a question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Binary decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+ " state[\"question\"]\n",
+ " filtered_documents = state[\"documents\"]\n",
+ "\n",
+ " if not filtered_documents:\n",
+ " # All documents have been filtered check_relevance\n",
+ " # We will re-generate a new query\n",
+ " print(\n",
+ " \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+ " )\n",
+ " return \"transform_query\"\n",
+ " else:\n",
+ " # We have relevant documents, so generate answer\n",
+ " print(\"---DECISION: GENERATE---\")\n",
+ " return \"generate\"\n",
+ "\n",
+ "\n",
+ "def grade_generation_v_documents_and_question(state):\n",
+ " \"\"\"\n",
+ " Determines whether the generation is grounded in the document and answers question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK HALLUCINATIONS---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ " generation = state[\"generation\"]\n",
+ "\n",
+ " score = hallucination_grader.invoke(\n",
+ " {\"documents\": documents, \"generation\": generation}\n",
+ " )\n",
+ " grade = score[\"score\"]\n",
+ "\n",
+ " # Check hallucination\n",
+ " if grade == \"yes\":\n",
+ " print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
+ " # Check question-answering\n",
+ " print(\"---GRADE GENERATION vs QUESTION---\")\n",
+ " score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
+ " grade = score[\"score\"]\n",
+ " if grade == \"yes\":\n",
+ " print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
+ " return \"useful\"\n",
+ " else:\n",
+ " print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
+ " return \"not useful\"\n",
+ " else:\n",
+ " pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
+ " return \"not supported\""
]
},
{
@@ -394,7 +776,20 @@
}
],
"source": [
- "from pprint import pprint\n\n# Run\ninputs = {\"question\": \"What is the AlphaCodium paper about?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"
+ "from pprint import pprint\n",
+ "\n",
+ "# Run\n",
+ "inputs = {\"question\": \"What is the AlphaCodium paper about?\"}\n",
+ "for output in app.stream(inputs):\n",
+ " for key, value in output.items():\n",
+ " # Node\n",
+ " pprint(f\"Node '{key}':\")\n",
+ " # Optional: print full state at each node\n",
+ " # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+ " pprint(\"\\n---\\n\")\n",
+ "\n",
+ "# Final generation\n",
+ "pprint(value[\"generation\"])"
]
},
{
@@ -406,16 +801,6 @@
"\n",
"https://smith.langchain.com/public/81813813-be53-403c-9877-afcd5786ca2e/r"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "4620ede9-b014-499f-8acf-80f80ce0d944",
- "metadata": {},
- "outputs": [],
- "source": [
- ""
- ]
}
],
"metadata": {
diff --git a/examples/rag/langgraph_crag.ipynb b/examples/rag/langgraph_crag.ipynb
index f131e398e..2e48143de 100644
--- a/examples/rag/langgraph_crag.ipynb
+++ b/examples/rag/langgraph_crag.ipynb
@@ -47,7 +47,9 @@
"id": "568c84d6-9df6-4b7b-b50d-476c0a64a04b",
"metadata": {},
"outputs": [],
- "source": ["! pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph tavily-python"]
+ "source": [
+ "! pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph tavily-python"
+ ]
},
{
"cell_type": "markdown",
@@ -63,7 +65,11 @@
"id": "74710419-158d-4270-931c-de83db7b580d",
"metadata": {},
"outputs": [],
- "source": ["import os\n\nos.environ[\"OPENAI_API_KEY\"] = \"\""]
+ "source": [
+ "import os\n",
+ "\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"\""
+ ]
},
{
"cell_type": "markdown",
@@ -81,7 +87,9 @@
"id": "c3ac6e65-2d4e-48dd-9fff-40047373332d",
"metadata": {},
"outputs": [],
- "source": ["os.environ[\"TAVILY_API_KEY\"] = \"\""]
+ "source": [
+ "os.environ[\"TAVILY_API_KEY\"] = \"\""
+ ]
},
{
"cell_type": "markdown",
@@ -99,7 +107,11 @@
"id": "e205f57e-5218-478b-ad8e-1723bdb0d45e",
"metadata": {},
"outputs": [],
- "source": ["os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"\""]
+ "source": [
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+ "os.environ[\"LANGCHAIN_API_KEY\"] = \"\""
+ ]
},
{
"cell_type": "markdown",
@@ -117,7 +129,34 @@
"id": "3a566a30-cf0e-4330-ad4d-9bf994bdfa86",
"metadata": {},
"outputs": [],
- "source": ["from langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_openai import OpenAIEmbeddings\n\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=250, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorDB\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=OpenAIEmbeddings(),\n)\nretriever = vectorstore.as_retriever()"]
+ "source": [
+ "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+ "from langchain_community.document_loaders import WebBaseLoader\n",
+ "from langchain_community.vectorstores import Chroma\n",
+ "from langchain_openai import OpenAIEmbeddings\n",
+ "\n",
+ "urls = [\n",
+ " \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+ "]\n",
+ "\n",
+ "docs = [WebBaseLoader(url).load() for url in urls]\n",
+ "docs_list = [item for sublist in docs for item in sublist]\n",
+ "\n",
+ "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+ " chunk_size=250, chunk_overlap=0\n",
+ ")\n",
+ "doc_splits = text_splitter.split_documents(docs_list)\n",
+ "\n",
+ "# Add to vectorDB\n",
+ "vectorstore = Chroma.from_documents(\n",
+ " documents=doc_splits,\n",
+ " collection_name=\"rag-chroma\",\n",
+ " embedding=OpenAIEmbeddings(),\n",
+ ")\n",
+ "retriever = vectorstore.as_retriever()"
+ ]
},
{
"cell_type": "markdown",
@@ -141,7 +180,44 @@
]
}
],
- "source": ["### Retrieval Grader\n\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\n\n# Data model\nclass GradeDocuments(BaseModel):\n \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n\n binary_score: str = Field(\n description=\"Documents are relevant to the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeDocuments)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\ngrade_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n ]\n)\n\nretrieval_grader = grade_prompt | structured_llm_grader\nquestion = \"agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"]
+ "source": [
+ "### Retrieval Grader\n",
+ "\n",
+ "from langchain_core.prompts import ChatPromptTemplate\n",
+ "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class GradeDocuments(BaseModel):\n",
+ " \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
+ "\n",
+ " binary_score: str = Field(\n",
+ " description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
+ " )\n",
+ "\n",
+ "\n",
+ "# LLM with function call\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+ " If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant. \\n\n",
+ " Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
+ "grade_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "retrieval_grader = grade_prompt | structured_llm_grader\n",
+ "question = \"agent memory\"\n",
+ "docs = retriever.get_relevant_documents(question)\n",
+ "doc_txt = docs[1].page_content\n",
+ "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+ ]
},
{
"cell_type": "code",
@@ -157,7 +233,31 @@
]
}
],
- "source": ["### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n\n\n# Post-processing\ndef format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"]
+ "source": [
+ "### Generate\n",
+ "\n",
+ "from langchain import hub\n",
+ "from langchain_core.output_parsers import StrOutputParser\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
+ "\n",
+ "\n",
+ "# Post-processing\n",
+ "def format_docs(docs):\n",
+ " return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+ "\n",
+ "\n",
+ "# Chain\n",
+ "rag_chain = prompt | llm | StrOutputParser()\n",
+ "\n",
+ "# Run\n",
+ "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+ "print(generation)"
+ ]
},
{
"cell_type": "code",
@@ -176,7 +276,28 @@
"output_type": "execute_result"
}
],
- "source": ["### Question Re-writer\n\n# LLM\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n\n# Prompt\nsystem = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n for web search. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\nre_write_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\n \"human\",\n \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n ),\n ]\n)\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nquestion_rewriter.invoke({\"question\": question})"]
+ "source": [
+ "### Question Re-writer\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+ " for web search. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
+ "re_write_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\n",
+ " \"human\",\n",
+ " \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
+ " ),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+ "question_rewriter.invoke({\"question\": question})"
+ ]
},
{
"cell_type": "markdown",
@@ -192,7 +313,13 @@
"id": "46d51b53-54a9-4e0a-9f14-e39998f5b340",
"metadata": {},
"outputs": [],
- "source": ["### Search\n\nfrom langchain_community.tools.tavily_search import TavilySearchResults\n\nweb_search_tool = TavilySearchResults(k=3)"]
+ "source": [
+ "### Search\n",
+ "\n",
+ "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+ "\n",
+ "web_search_tool = TavilySearchResults(k=3)"
+ ]
},
{
"cell_type": "markdown",
@@ -212,7 +339,28 @@
"id": "94b3945f-ef0f-458d-a443-f763903550b0",
"metadata": {},
"outputs": [],
- "source": ["from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n web_search: whether to add search\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n web_search: str\n documents: List[str]"]
+ "source": [
+ "from typing import List\n",
+ "\n",
+ "from typing_extensions import TypedDict\n",
+ "\n",
+ "\n",
+ "class GraphState(TypedDict):\n",
+ " \"\"\"\n",
+ " Represents the state of our graph.\n",
+ "\n",
+ " Attributes:\n",
+ " question: question\n",
+ " generation: LLM generation\n",
+ " web_search: whether to add search\n",
+ " documents: list of documents\n",
+ " \"\"\"\n",
+ "\n",
+ " question: str\n",
+ " generation: str\n",
+ " web_search: str\n",
+ " documents: List[str]"
+ ]
},
{
"cell_type": "code",
@@ -220,7 +368,155 @@
"id": "efd639c5-82e2-45e6-a94a-6a4039646ef5",
"metadata": {},
"outputs": [],
- "source": ["from langchain.schema import Document\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.get_relevant_documents(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n web_search = \"No\"\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n web_search = \"Yes\"\n continue\n return {\"documents\": filtered_docs, \"question\": question, \"web_search\": web_search}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}\n\n\ndef web_search(state):\n \"\"\"\n Web search based on the re-phrased question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with appended web results\n \"\"\"\n\n print(\"---WEB SEARCH---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Web search\n docs = web_search_tool.invoke({\"query\": question})\n web_results = \"\\n\".join([d[\"content\"] for d in docs])\n web_results = Document(page_content=web_results)\n documents.append(web_results)\n\n return {\"documents\": documents, \"question\": question}\n\n\n### Edges\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n web_search = state[\"web_search\"]\n state[\"documents\"]\n\n if web_search == \"Yes\":\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\""]
+ "source": [
+ "from langchain.schema import Document\n",
+ "\n",
+ "\n",
+ "def retrieve(state):\n",
+ " \"\"\"\n",
+ " Retrieve documents\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, documents, that contains retrieved documents\n",
+ " \"\"\"\n",
+ " print(\"---RETRIEVE---\")\n",
+ " question = state[\"question\"]\n",
+ "\n",
+ " # Retrieval\n",
+ " documents = retriever.get_relevant_documents(question)\n",
+ " return {\"documents\": documents, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def generate(state):\n",
+ " \"\"\"\n",
+ " Generate answer\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, generation, that contains LLM generation\n",
+ " \"\"\"\n",
+ " print(\"---GENERATE---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # RAG generation\n",
+ " generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+ " return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+ "\n",
+ "\n",
+ "def grade_documents(state):\n",
+ " \"\"\"\n",
+ " Determines whether the retrieved documents are relevant to the question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with only filtered relevant documents\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Score each doc\n",
+ " filtered_docs = []\n",
+ " web_search = \"No\"\n",
+ " for d in documents:\n",
+ " score = retrieval_grader.invoke(\n",
+ " {\"question\": question, \"document\": d.page_content}\n",
+ " )\n",
+ " grade = score.binary_score\n",
+ " if grade == \"yes\":\n",
+ " print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+ " filtered_docs.append(d)\n",
+ " else:\n",
+ " print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+ " web_search = \"Yes\"\n",
+ " continue\n",
+ " return {\"documents\": filtered_docs, \"question\": question, \"web_search\": web_search}\n",
+ "\n",
+ "\n",
+ "def transform_query(state):\n",
+ " \"\"\"\n",
+ " Transform the query to produce a better question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates question key with a re-phrased question\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---TRANSFORM QUERY---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Re-write question\n",
+ " better_question = question_rewriter.invoke({\"question\": question})\n",
+ " return {\"documents\": documents, \"question\": better_question}\n",
+ "\n",
+ "\n",
+ "def web_search(state):\n",
+ " \"\"\"\n",
+ " Web search based on the re-phrased question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with appended web results\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---WEB SEARCH---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Web search\n",
+ " docs = web_search_tool.invoke({\"query\": question})\n",
+ " web_results = \"\\n\".join([d[\"content\"] for d in docs])\n",
+ " web_results = Document(page_content=web_results)\n",
+ " documents.append(web_results)\n",
+ "\n",
+ " return {\"documents\": documents, \"question\": question}\n",
+ "\n",
+ "\n",
+ "### Edges\n",
+ "\n",
+ "\n",
+ "def decide_to_generate(state):\n",
+ " \"\"\"\n",
+ " Determines whether to generate an answer, or re-generate a question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Binary decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+ " state[\"question\"]\n",
+ " web_search = state[\"web_search\"]\n",
+ " state[\"documents\"]\n",
+ "\n",
+ " if web_search == \"Yes\":\n",
+ " # All documents have been filtered check_relevance\n",
+ " # We will re-generate a new query\n",
+ " print(\n",
+ " \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+ " )\n",
+ " return \"transform_query\"\n",
+ " else:\n",
+ " # We have relevant documents, so generate answer\n",
+ " print(\"---DECISION: GENERATE---\")\n",
+ " return \"generate\""
+ ]
},
{
"cell_type": "markdown",
@@ -238,7 +534,36 @@
"id": "dedae17a-98c6-474d-90a7-9234b7c8cea0",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"transform_query\", transform_query) # transform_query\nworkflow.add_node(\"web_search_node\", web_search) # web search\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"transform_query\": \"transform_query\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"transform_query\", \"web_search_node\")\nworkflow.add_edge(\"web_search_node\", \"generate\")\nworkflow.add_edge(\"generate\", END)\n\n# Compile\napp = workflow.compile()"]
+ "source": [
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "workflow = StateGraph(GraphState)\n",
+ "\n",
+ "# Define the nodes\n",
+ "workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
+ "workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
+ "workflow.add_node(\"generate\", generate) # generatae\n",
+ "workflow.add_node(\"transform_query\", transform_query) # transform_query\n",
+ "workflow.add_node(\"web_search_node\", web_search) # web search\n",
+ "\n",
+ "# Build graph\n",
+ "workflow.add_edge(START, \"retrieve\")\n",
+ "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+ "workflow.add_conditional_edges(\n",
+ " \"grade_documents\",\n",
+ " decide_to_generate,\n",
+ " {\n",
+ " \"transform_query\": \"transform_query\",\n",
+ " \"generate\": \"generate\",\n",
+ " },\n",
+ ")\n",
+ "workflow.add_edge(\"transform_query\", \"web_search_node\")\n",
+ "workflow.add_edge(\"web_search_node\", \"generate\")\n",
+ "workflow.add_edge(\"generate\", END)\n",
+ "\n",
+ "# Compile\n",
+ "app = workflow.compile()"
+ ]
},
{
"cell_type": "code",
@@ -281,7 +606,22 @@
]
}
],
- "source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"What are the types of agent memory?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
+ "source": [
+ "from pprint import pprint\n",
+ "\n",
+ "# Run\n",
+ "inputs = {\"question\": \"What are the types of agent memory?\"}\n",
+ "for output in app.stream(inputs):\n",
+ " for key, value in output.items():\n",
+ " # Node\n",
+ " pprint(f\"Node '{key}':\")\n",
+ " # Optional: print full state at each node\n",
+ " # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+ " pprint(\"\\n---\\n\")\n",
+ "\n",
+ "# Final generation\n",
+ "pprint(value[\"generation\"])"
+ ]
},
{
"cell_type": "code",
@@ -326,7 +666,22 @@
]
}
],
- "source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"How does the AlphaCodium paper work?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
+ "source": [
+ "from pprint import pprint\n",
+ "\n",
+ "# Run\n",
+ "inputs = {\"question\": \"How does the AlphaCodium paper work?\"}\n",
+ "for output in app.stream(inputs):\n",
+ " for key, value in output.items():\n",
+ " # Node\n",
+ " pprint(f\"Node '{key}':\")\n",
+ " # Optional: print full state at each node\n",
+ " # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+ " pprint(\"\\n---\\n\")\n",
+ "\n",
+ "# Final generation\n",
+ "pprint(value[\"generation\"])"
+ ]
},
{
"cell_type": "markdown",
@@ -339,14 +694,6 @@
"\n",
"* https://smith.langchain.com/public/497c8ed9-d9e2-429e-8ada-e64de3ec26c9/r"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "6ce65be5-fd12-4ffc-984c-34c132693e69",
- "metadata": {},
- "outputs": [],
- "source": [""]
}
],
"metadata": {
diff --git a/examples/rag/langgraph_crag_local.ipynb b/examples/rag/langgraph_crag_local.ipynb
index d6e791ae3..b43a89929 100644
--- a/examples/rag/langgraph_crag_local.ipynb
+++ b/examples/rag/langgraph_crag_local.ipynb
@@ -57,7 +57,8 @@
"metadata": {},
"outputs": [],
"source": [
- "%%capture --no-stderr\n%pip install -U langchain_community tiktoken langchainhub scikit-learn langchain langgraph tavily-python nomic[local] langchain-nomic langchain_openai"
+ "%%capture --no-stderr\n",
+ "%pip install -U langchain_community tiktoken langchainhub scikit-learn langchain langgraph tavily-python nomic[local] langchain-nomic langchain_openai"
]
},
{
@@ -80,7 +81,8 @@
"metadata": {},
"outputs": [],
"source": [
- "# Embedding (optional)\nos.environ[\"OPENAI_API_KEY\"] = \"xxx\""
+ "# Embedding (optional)\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"xxx\""
]
},
{
@@ -90,7 +92,11 @@
"metadata": {},
"outputs": [],
"source": [
- "# Tracing and testing (optional)\nos.environ[\"LANGCHAIN_API_KEY\"] = \"xxx\"\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_PROJECT\"] = \"corrective-rag-agent-testing\""
+ "# Tracing and testing (optional)\n",
+ "os.environ[\"LANGCHAIN_API_KEY\"] = \"xxx\"\n",
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+ "os.environ[\"LANGCHAIN_PROJECT\"] = \"corrective-rag-agent-testing\""
]
},
{
@@ -110,7 +116,9 @@
"metadata": {},
"outputs": [],
"source": [
- "local_llm = \"llama3\"\nmodel_tested = \"llama3-8b\"\nmetadata = f\"CRAG, {model_tested}\""
+ "local_llm = \"llama3\"\n",
+ "model_tested = \"llama3-8b\"\n",
+ "metadata = f\"CRAG, {model_tested}\""
]
},
{
@@ -204,7 +212,45 @@
}
],
"source": [
- "### Retrieval Grader\n\nfrom langchain.prompts import PromptTemplate\nfrom langchain_community.chat_models import ChatOllama\nfrom langchain_core.output_parsers import JsonOutputParser\nfrom langchain_mistralai.chat_models import ChatMistralAI\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are a teacher grading a quiz. You will be given: \n 1/ a QUESTION\n 2/ A FACT provided by the student\n \n You are grading RELEVANCE RECALL:\n A score of 1 means that ANY of the statements in the FACT are relevant to the QUESTION. \n A score of 0 means that NONE of the statements in the FACT are relevant to the QUESTION. \n 1 is the highest (best) score. 0 is the lowest score you can give. \n \n Explain your reasoning in a step-by-step manner. Ensure your reasoning and conclusion are correct. \n \n Avoid simply stating the correct answer at the outset.\n \n Question: {question} \\n\n Fact: \\n\\n {documents} \\n\\n\n \n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\n \"\"\",\n input_variables=[\"question\", \"documents\"],\n)\n\nretrieval_grader = prompt | llm | JsonOutputParser()\nquestion = \"agent memory\"\ndocs = retriever.invoke(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"documents\": doc_txt}))"
+ "### Retrieval Grader\n",
+ "\n",
+ "from langchain.prompts import PromptTemplate\n",
+ "from langchain_community.chat_models import ChatOllama\n",
+ "from langchain_core.output_parsers import JsonOutputParser\n",
+ "from langchain_mistralai.chat_models import ChatMistralAI\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are a teacher grading a quiz. You will be given: \n",
+ " 1/ a QUESTION\n",
+ " 2/ A FACT provided by the student\n",
+ " \n",
+ " You are grading RELEVANCE RECALL:\n",
+ " A score of 1 means that ANY of the statements in the FACT are relevant to the QUESTION. \n",
+ " A score of 0 means that NONE of the statements in the FACT are relevant to the QUESTION. \n",
+ " 1 is the highest (best) score. 0 is the lowest score you can give. \n",
+ " \n",
+ " Explain your reasoning in a step-by-step manner. Ensure your reasoning and conclusion are correct. \n",
+ " \n",
+ " Avoid simply stating the correct answer at the outset.\n",
+ " \n",
+ " Question: {question} \\n\n",
+ " Fact: \\n\\n {documents} \\n\\n\n",
+ " \n",
+ " Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n",
+ " Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\n",
+ " \"\"\",\n",
+ " input_variables=[\"question\", \"documents\"],\n",
+ ")\n",
+ "\n",
+ "retrieval_grader = prompt | llm | JsonOutputParser()\n",
+ "question = \"agent memory\"\n",
+ "docs = retriever.invoke(question)\n",
+ "doc_txt = docs[1].page_content\n",
+ "print(retrieval_grader.invoke({\"question\": question, \"documents\": doc_txt}))"
]
},
{
@@ -222,7 +268,35 @@
}
],
"source": [
- "### Generate\n\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are an assistant for question-answering tasks. \n \n Use the following documents to answer the question. \n \n If you don't know the answer, just say that you don't know. \n \n Use three sentences maximum and keep the answer concise:\n Question: {question} \n Documents: {documents} \n Answer: \n \"\"\",\n input_variables=[\"question\", \"documents\"],\n)\n\n# LLM\nllm = ChatOllama(model=local_llm, temperature=0)\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"documents\": docs, \"question\": question})\nprint(generation)"
+ "### Generate\n",
+ "\n",
+ "from langchain_core.output_parsers import StrOutputParser\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are an assistant for question-answering tasks. \n",
+ " \n",
+ " Use the following documents to answer the question. \n",
+ " \n",
+ " If you don't know the answer, just say that you don't know. \n",
+ " \n",
+ " Use three sentences maximum and keep the answer concise:\n",
+ " Question: {question} \n",
+ " Documents: {documents} \n",
+ " Answer: \n",
+ " \"\"\",\n",
+ " input_variables=[\"question\", \"documents\"],\n",
+ ")\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, temperature=0)\n",
+ "\n",
+ "# Chain\n",
+ "rag_chain = prompt | llm | StrOutputParser()\n",
+ "\n",
+ "# Run\n",
+ "generation = rag_chain.invoke({\"documents\": docs, \"question\": question})\n",
+ "print(generation)"
]
},
{
@@ -232,7 +306,11 @@
"metadata": {},
"outputs": [],
"source": [
- "### Search\n\nfrom langchain_community.tools.tavily_search import TavilySearchResults\n\nweb_search_tool = TavilySearchResults(k=3)"
+ "### Search\n",
+ "\n",
+ "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+ "\n",
+ "web_search_tool = TavilySearchResults(k=3)"
]
},
{
@@ -263,7 +341,175 @@
}
],
"source": [
- "from typing import List\nfrom typing_extensions import TypedDict\nfrom IPython.display import Image, display\nfrom langchain.schema import Document\nfrom langgraph.graph import START, END, StateGraph\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n search: whether to add search\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n search: str\n documents: List[str]\n steps: List[str]\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n question = state[\"question\"]\n documents = retriever.invoke(question)\n steps = state[\"steps\"]\n steps.append(\"retrieve_documents\")\n return {\"documents\": documents, \"question\": question, \"steps\": steps}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = rag_chain.invoke({\"documents\": documents, \"question\": question})\n steps = state[\"steps\"]\n steps.append(\"generate_answer\")\n return {\n \"documents\": documents,\n \"question\": question,\n \"generation\": generation,\n \"steps\": steps,\n }\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n question = state[\"question\"]\n documents = state[\"documents\"]\n steps = state[\"steps\"]\n steps.append(\"grade_document_retrieval\")\n filtered_docs = []\n search = \"No\"\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"documents\": d.page_content}\n )\n grade = score[\"score\"]\n if grade == \"yes\":\n filtered_docs.append(d)\n else:\n search = \"Yes\"\n continue\n return {\n \"documents\": filtered_docs,\n \"question\": question,\n \"search\": search,\n \"steps\": steps,\n }\n\n\ndef web_search(state):\n \"\"\"\n Web search based on the re-phrased question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with appended web results\n \"\"\"\n\n question = state[\"question\"]\n documents = state.get(\"documents\", [])\n steps = state[\"steps\"]\n steps.append(\"web_search\")\n web_results = web_search_tool.invoke({\"query\": question})\n documents.extend(\n [\n Document(page_content=d[\"content\"], metadata={\"url\": d[\"url\"]})\n for d in web_results\n ]\n )\n return {\"documents\": documents, \"question\": question, \"steps\": steps}\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n search = state[\"search\"]\n if search == \"Yes\":\n return \"search\"\n else:\n return \"generate\"\n\n\n# Graph\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"web_search\", web_search) # web search\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"search\": \"web_search\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"web_search\", \"generate\")\nworkflow.add_edge(\"generate\", END)\n\ncustom_graph = workflow.compile()\n\ndisplay(Image(custom_graph.get_graph(xray=True).draw_mermaid_png()))"
+ "from typing import List\n",
+ "from typing_extensions import TypedDict\n",
+ "from IPython.display import Image, display\n",
+ "from langchain.schema import Document\n",
+ "from langgraph.graph import START, END, StateGraph\n",
+ "\n",
+ "\n",
+ "class GraphState(TypedDict):\n",
+ " \"\"\"\n",
+ " Represents the state of our graph.\n",
+ "\n",
+ " Attributes:\n",
+ " question: question\n",
+ " generation: LLM generation\n",
+ " search: whether to add search\n",
+ " documents: list of documents\n",
+ " \"\"\"\n",
+ "\n",
+ " question: str\n",
+ " generation: str\n",
+ " search: str\n",
+ " documents: List[str]\n",
+ " steps: List[str]\n",
+ "\n",
+ "\n",
+ "def retrieve(state):\n",
+ " \"\"\"\n",
+ " Retrieve documents\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, documents, that contains retrieved documents\n",
+ " \"\"\"\n",
+ " question = state[\"question\"]\n",
+ " documents = retriever.invoke(question)\n",
+ " steps = state[\"steps\"]\n",
+ " steps.append(\"retrieve_documents\")\n",
+ " return {\"documents\": documents, \"question\": question, \"steps\": steps}\n",
+ "\n",
+ "\n",
+ "def generate(state):\n",
+ " \"\"\"\n",
+ " Generate answer\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, generation, that contains LLM generation\n",
+ " \"\"\"\n",
+ "\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ " generation = rag_chain.invoke({\"documents\": documents, \"question\": question})\n",
+ " steps = state[\"steps\"]\n",
+ " steps.append(\"generate_answer\")\n",
+ " return {\n",
+ " \"documents\": documents,\n",
+ " \"question\": question,\n",
+ " \"generation\": generation,\n",
+ " \"steps\": steps,\n",
+ " }\n",
+ "\n",
+ "\n",
+ "def grade_documents(state):\n",
+ " \"\"\"\n",
+ " Determines whether the retrieved documents are relevant to the question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with only filtered relevant documents\n",
+ " \"\"\"\n",
+ "\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ " steps = state[\"steps\"]\n",
+ " steps.append(\"grade_document_retrieval\")\n",
+ " filtered_docs = []\n",
+ " search = \"No\"\n",
+ " for d in documents:\n",
+ " score = retrieval_grader.invoke(\n",
+ " {\"question\": question, \"documents\": d.page_content}\n",
+ " )\n",
+ " grade = score[\"score\"]\n",
+ " if grade == \"yes\":\n",
+ " filtered_docs.append(d)\n",
+ " else:\n",
+ " search = \"Yes\"\n",
+ " continue\n",
+ " return {\n",
+ " \"documents\": filtered_docs,\n",
+ " \"question\": question,\n",
+ " \"search\": search,\n",
+ " \"steps\": steps,\n",
+ " }\n",
+ "\n",
+ "\n",
+ "def web_search(state):\n",
+ " \"\"\"\n",
+ " Web search based on the re-phrased question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with appended web results\n",
+ " \"\"\"\n",
+ "\n",
+ " question = state[\"question\"]\n",
+ " documents = state.get(\"documents\", [])\n",
+ " steps = state[\"steps\"]\n",
+ " steps.append(\"web_search\")\n",
+ " web_results = web_search_tool.invoke({\"query\": question})\n",
+ " documents.extend(\n",
+ " [\n",
+ " Document(page_content=d[\"content\"], metadata={\"url\": d[\"url\"]})\n",
+ " for d in web_results\n",
+ " ]\n",
+ " )\n",
+ " return {\"documents\": documents, \"question\": question, \"steps\": steps}\n",
+ "\n",
+ "\n",
+ "def decide_to_generate(state):\n",
+ " \"\"\"\n",
+ " Determines whether to generate an answer, or re-generate a question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Binary decision for next node to call\n",
+ " \"\"\"\n",
+ " search = state[\"search\"]\n",
+ " if search == \"Yes\":\n",
+ " return \"search\"\n",
+ " else:\n",
+ " return \"generate\"\n",
+ "\n",
+ "\n",
+ "# Graph\n",
+ "workflow = StateGraph(GraphState)\n",
+ "\n",
+ "# Define the nodes\n",
+ "workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
+ "workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
+ "workflow.add_node(\"generate\", generate) # generatae\n",
+ "workflow.add_node(\"web_search\", web_search) # web search\n",
+ "\n",
+ "# Build graph\n",
+ "workflow.add_edge(START, \"retrieve\")\n",
+ "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+ "workflow.add_conditional_edges(\n",
+ " \"grade_documents\",\n",
+ " decide_to_generate,\n",
+ " {\n",
+ " \"search\": \"web_search\",\n",
+ " \"generate\": \"generate\",\n",
+ " },\n",
+ ")\n",
+ "workflow.add_edge(\"web_search\", \"generate\")\n",
+ "workflow.add_edge(\"generate\", END)\n",
+ "\n",
+ "custom_graph = workflow.compile()\n",
+ "\n",
+ "display(Image(custom_graph.get_graph(xray=True).draw_mermaid_png()))"
]
},
{
@@ -339,7 +585,39 @@
"metadata": {},
"outputs": [],
"source": [
- "from langsmith import Client\n\nclient = Client()\n\n# Create a dataset\nexamples = [\n (\n \"How does the ReAct agent use self-reflection? \",\n \"ReAct integrates reasoning and acting, performing actions - such tools like Wikipedia search API - and then observing / reasoning about the tool outputs.\",\n ),\n (\n \"What are the types of biases that can arise with few-shot prompting?\",\n \"The biases that can arise with few-shot prompting include (1) Majority label bias, (2) Recency bias, and (3) Common token bias.\",\n ),\n (\n \"What are five types of adversarial attacks?\",\n \"Five types of adversarial attacks are (1) Token manipulation, (2) Gradient based attack, (3) Jailbreak prompting, (4) Human red-teaming, (5) Model red-teaming.\",\n ),\n (\n \"Who did the Chicago Bears draft first in the 2024 NFL draft”?\",\n \"The Chicago Bears drafted Caleb Williams first in the 2024 NFL draft.\",\n ),\n (\"Who won the 2024 NBA finals?\", \"The Boston Celtics on the 2024 NBA finals\"),\n]\n\n# Save it\ndataset_name = \"Corrective RAG Agent Testing\"\nif not client.has_dataset(dataset_name=dataset_name):\n dataset = client.create_dataset(dataset_name=dataset_name)\n inputs, outputs = zip(\n *[({\"input\": text}, {\"output\": label}) for text, label in examples]\n )\n client.create_examples(inputs=inputs, outputs=outputs, dataset_id=dataset.id)"
+ "from langsmith import Client\n",
+ "\n",
+ "client = Client()\n",
+ "\n",
+ "# Create a dataset\n",
+ "examples = [\n",
+ " (\n",
+ " \"How does the ReAct agent use self-reflection? \",\n",
+ " \"ReAct integrates reasoning and acting, performing actions - such tools like Wikipedia search API - and then observing / reasoning about the tool outputs.\",\n",
+ " ),\n",
+ " (\n",
+ " \"What are the types of biases that can arise with few-shot prompting?\",\n",
+ " \"The biases that can arise with few-shot prompting include (1) Majority label bias, (2) Recency bias, and (3) Common token bias.\",\n",
+ " ),\n",
+ " (\n",
+ " \"What are five types of adversarial attacks?\",\n",
+ " \"Five types of adversarial attacks are (1) Token manipulation, (2) Gradient based attack, (3) Jailbreak prompting, (4) Human red-teaming, (5) Model red-teaming.\",\n",
+ " ),\n",
+ " (\n",
+ " \"Who did the Chicago Bears draft first in the 2024 NFL draft”?\",\n",
+ " \"The Chicago Bears drafted Caleb Williams first in the 2024 NFL draft.\",\n",
+ " ),\n",
+ " (\"Who won the 2024 NBA finals?\", \"The Boston Celtics on the 2024 NBA finals\"),\n",
+ "]\n",
+ "\n",
+ "# Save it\n",
+ "dataset_name = \"Corrective RAG Agent Testing\"\n",
+ "if not client.has_dataset(dataset_name=dataset_name):\n",
+ " dataset = client.create_dataset(dataset_name=dataset_name)\n",
+ " inputs, outputs = zip(\n",
+ " *[({\"input\": text}, {\"output\": label}) for text, label in examples]\n",
+ " )\n",
+ " client.create_examples(inputs=inputs, outputs=outputs, dataset_id=dataset.id)"
]
},
{
@@ -551,16 +829,6 @@
"\n",
"However, the answer accuracy performance lags the larger models with `custom agent` implementations."
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "79295798-0181-417e-abad-11dddb6ff05e",
- "metadata": {},
- "outputs": [],
- "source": [
- ""
- ]
}
],
"metadata": {
diff --git a/examples/rag/langgraph_self_rag.ipynb b/examples/rag/langgraph_self_rag.ipynb
index cb2d5d934..496237443 100644
--- a/examples/rag/langgraph_self_rag.ipynb
+++ b/examples/rag/langgraph_self_rag.ipynb
@@ -59,7 +59,9 @@
"id": "a384cc48-0425-4e8f-aafc-cfb8e56025c9",
"metadata": {},
"outputs": [],
- "source": ["! pip install -U langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph"]
+ "source": [
+ "! pip install -U langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph"
+ ]
},
{
"cell_type": "markdown",
@@ -75,7 +77,11 @@
"id": "f18b63c7-d0d3-41c1-ae6b-5a0f1b8ccf0f",
"metadata": {},
"outputs": [],
- "source": ["import os\n\nos.environ[\"OPENAI_API_KEY\"] = \"\""]
+ "source": [
+ "import os\n",
+ "\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"\""
+ ]
},
{
"cell_type": "markdown",
@@ -93,7 +99,11 @@
"id": "ccc3dae5-1df6-48ca-af8a-50f0e6128876",
"metadata": {},
"outputs": [],
- "source": ["os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"\""]
+ "source": [
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+ "os.environ[\"LANGCHAIN_API_KEY\"] = \"\""
+ ]
},
{
"cell_type": "markdown",
@@ -111,7 +121,34 @@
"id": "565a6d44-2c9f-4fff-b1ec-eea05df9350d",
"metadata": {},
"outputs": [],
- "source": ["from langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_openai import OpenAIEmbeddings\n\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=250, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorDB\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=OpenAIEmbeddings(),\n)\nretriever = vectorstore.as_retriever()"]
+ "source": [
+ "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+ "from langchain_community.document_loaders import WebBaseLoader\n",
+ "from langchain_community.vectorstores import Chroma\n",
+ "from langchain_openai import OpenAIEmbeddings\n",
+ "\n",
+ "urls = [\n",
+ " \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+ "]\n",
+ "\n",
+ "docs = [WebBaseLoader(url).load() for url in urls]\n",
+ "docs_list = [item for sublist in docs for item in sublist]\n",
+ "\n",
+ "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+ " chunk_size=250, chunk_overlap=0\n",
+ ")\n",
+ "doc_splits = text_splitter.split_documents(docs_list)\n",
+ "\n",
+ "# Add to vectorDB\n",
+ "vectorstore = Chroma.from_documents(\n",
+ " documents=doc_splits,\n",
+ " collection_name=\"rag-chroma\",\n",
+ " embedding=OpenAIEmbeddings(),\n",
+ ")\n",
+ "retriever = vectorstore.as_retriever()"
+ ]
},
{
"cell_type": "markdown",
@@ -143,7 +180,46 @@
]
}
],
- "source": ["### Retrieval Grader\n\n\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\n\n# Data model\nclass GradeDocuments(BaseModel):\n \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n\n binary_score: str = Field(\n description=\"Documents are relevant to the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeDocuments)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\ngrade_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n ]\n)\n\nretrieval_grader = grade_prompt | structured_llm_grader\nquestion = \"agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"]
+ "source": [
+ "### Retrieval Grader\n",
+ "\n",
+ "\n",
+ "from langchain_core.prompts import ChatPromptTemplate\n",
+ "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+ "from langchain_openai import ChatOpenAI\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class GradeDocuments(BaseModel):\n",
+ " \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
+ "\n",
+ " binary_score: str = Field(\n",
+ " description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
+ " )\n",
+ "\n",
+ "\n",
+ "# LLM with function call\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+ " It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
+ " If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
+ " Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
+ "grade_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "retrieval_grader = grade_prompt | structured_llm_grader\n",
+ "question = \"agent memory\"\n",
+ "docs = retriever.get_relevant_documents(question)\n",
+ "doc_txt = docs[1].page_content\n",
+ "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+ ]
},
{
"cell_type": "code",
@@ -159,7 +235,31 @@
]
}
],
- "source": ["### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n\n\n# Post-processing\ndef format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"]
+ "source": [
+ "### Generate\n",
+ "\n",
+ "from langchain import hub\n",
+ "from langchain_core.output_parsers import StrOutputParser\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
+ "\n",
+ "\n",
+ "# Post-processing\n",
+ "def format_docs(docs):\n",
+ " return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+ "\n",
+ "\n",
+ "# Chain\n",
+ "rag_chain = prompt | llm | StrOutputParser()\n",
+ "\n",
+ "# Run\n",
+ "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+ "print(generation)"
+ ]
},
{
"cell_type": "code",
@@ -178,7 +278,36 @@
"output_type": "execute_result"
}
],
- "source": ["### Hallucination Grader\n\n\n# Data model\nclass GradeHallucinations(BaseModel):\n \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n\n binary_score: str = Field(\n description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeHallucinations)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n \n Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\nhallucination_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n ]\n)\n\nhallucination_grader = hallucination_prompt | structured_llm_grader\nhallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"]
+ "source": [
+ "### Hallucination Grader\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class GradeHallucinations(BaseModel):\n",
+ " \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n",
+ "\n",
+ " binary_score: str = Field(\n",
+ " description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n",
+ " )\n",
+ "\n",
+ "\n",
+ "# LLM with function call\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "structured_llm_grader = llm.with_structured_output(GradeHallucinations)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n \n",
+ " Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\n",
+ "hallucination_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "hallucination_grader = hallucination_prompt | structured_llm_grader\n",
+ "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+ ]
},
{
"cell_type": "code",
@@ -197,7 +326,36 @@
"output_type": "execute_result"
}
],
- "source": ["### Answer Grader\n\n\n# Data model\nclass GradeAnswer(BaseModel):\n \"\"\"Binary score to assess answer addresses question.\"\"\"\n\n binary_score: str = Field(\n description=\"Answer addresses the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeAnswer)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n \n Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\nanswer_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n ]\n)\n\nanswer_grader = answer_prompt | structured_llm_grader\nanswer_grader.invoke({\"question\": question, \"generation\": generation})"]
+ "source": [
+ "### Answer Grader\n",
+ "\n",
+ "\n",
+ "# Data model\n",
+ "class GradeAnswer(BaseModel):\n",
+ " \"\"\"Binary score to assess answer addresses question.\"\"\"\n",
+ "\n",
+ " binary_score: str = Field(\n",
+ " description=\"Answer addresses the question, 'yes' or 'no'\"\n",
+ " )\n",
+ "\n",
+ "\n",
+ "# LLM with function call\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "structured_llm_grader = llm.with_structured_output(GradeAnswer)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n \n",
+ " Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\n",
+ "answer_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "answer_grader = answer_prompt | structured_llm_grader\n",
+ "answer_grader.invoke({\"question\": question, \"generation\": generation})"
+ ]
},
{
"cell_type": "code",
@@ -216,7 +374,28 @@
"output_type": "execute_result"
}
],
- "source": ["### Question Re-writer\n\n# LLM\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n\n# Prompt\nsystem = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\nre_write_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\n \"human\",\n \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n ),\n ]\n)\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nquestion_rewriter.invoke({\"question\": question})"]
+ "source": [
+ "### Question Re-writer\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+ " for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
+ "re_write_prompt = ChatPromptTemplate.from_messages(\n",
+ " [\n",
+ " (\"system\", system),\n",
+ " (\n",
+ " \"human\",\n",
+ " \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
+ " ),\n",
+ " ]\n",
+ ")\n",
+ "\n",
+ "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+ "question_rewriter.invoke({\"question\": question})"
+ ]
},
{
"cell_type": "markdown",
@@ -236,7 +415,26 @@
"id": "f1617e9e-66a8-4c1a-a1fe-cc936284c085",
"metadata": {},
"outputs": [],
- "source": ["from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n documents: List[str]"]
+ "source": [
+ "from typing import List\n",
+ "\n",
+ "from typing_extensions import TypedDict\n",
+ "\n",
+ "\n",
+ "class GraphState(TypedDict):\n",
+ " \"\"\"\n",
+ " Represents the state of our graph.\n",
+ "\n",
+ " Attributes:\n",
+ " question: question\n",
+ " generation: LLM generation\n",
+ " documents: list of documents\n",
+ " \"\"\"\n",
+ "\n",
+ " question: str\n",
+ " generation: str\n",
+ " documents: List[str]"
+ ]
},
{
"cell_type": "code",
@@ -244,7 +442,167 @@
"id": "add509d8-6682-4127-8d95-13dd37d79702",
"metadata": {},
"outputs": [],
- "source": ["### Nodes\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.get_relevant_documents(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n continue\n return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}\n\n\n### Edges\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n filtered_documents = state[\"documents\"]\n\n if not filtered_documents:\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\"\n\n\ndef grade_generation_v_documents_and_question(state):\n \"\"\"\n Determines whether the generation is grounded in the document and answers question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Decision for next node to call\n \"\"\"\n\n print(\"---CHECK HALLUCINATIONS---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = state[\"generation\"]\n\n score = hallucination_grader.invoke(\n {\"documents\": documents, \"generation\": generation}\n )\n grade = score.binary_score\n\n # Check hallucination\n if grade == \"yes\":\n print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n # Check question-answering\n print(\"---GRADE GENERATION vs QUESTION---\")\n score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n return \"useful\"\n else:\n print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n return \"not useful\"\n else:\n pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n return \"not supported\""]
+ "source": [
+ "### Nodes\n",
+ "\n",
+ "\n",
+ "def retrieve(state):\n",
+ " \"\"\"\n",
+ " Retrieve documents\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, documents, that contains retrieved documents\n",
+ " \"\"\"\n",
+ " print(\"---RETRIEVE---\")\n",
+ " question = state[\"question\"]\n",
+ "\n",
+ " # Retrieval\n",
+ " documents = retriever.get_relevant_documents(question)\n",
+ " return {\"documents\": documents, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def generate(state):\n",
+ " \"\"\"\n",
+ " Generate answer\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, generation, that contains LLM generation\n",
+ " \"\"\"\n",
+ " print(\"---GENERATE---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # RAG generation\n",
+ " generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+ " return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+ "\n",
+ "\n",
+ "def grade_documents(state):\n",
+ " \"\"\"\n",
+ " Determines whether the retrieved documents are relevant to the question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with only filtered relevant documents\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Score each doc\n",
+ " filtered_docs = []\n",
+ " for d in documents:\n",
+ " score = retrieval_grader.invoke(\n",
+ " {\"question\": question, \"document\": d.page_content}\n",
+ " )\n",
+ " grade = score.binary_score\n",
+ " if grade == \"yes\":\n",
+ " print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+ " filtered_docs.append(d)\n",
+ " else:\n",
+ " print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+ " continue\n",
+ " return {\"documents\": filtered_docs, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def transform_query(state):\n",
+ " \"\"\"\n",
+ " Transform the query to produce a better question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates question key with a re-phrased question\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---TRANSFORM QUERY---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Re-write question\n",
+ " better_question = question_rewriter.invoke({\"question\": question})\n",
+ " return {\"documents\": documents, \"question\": better_question}\n",
+ "\n",
+ "\n",
+ "### Edges\n",
+ "\n",
+ "\n",
+ "def decide_to_generate(state):\n",
+ " \"\"\"\n",
+ " Determines whether to generate an answer, or re-generate a question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Binary decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+ " state[\"question\"]\n",
+ " filtered_documents = state[\"documents\"]\n",
+ "\n",
+ " if not filtered_documents:\n",
+ " # All documents have been filtered check_relevance\n",
+ " # We will re-generate a new query\n",
+ " print(\n",
+ " \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+ " )\n",
+ " return \"transform_query\"\n",
+ " else:\n",
+ " # We have relevant documents, so generate answer\n",
+ " print(\"---DECISION: GENERATE---\")\n",
+ " return \"generate\"\n",
+ "\n",
+ "\n",
+ "def grade_generation_v_documents_and_question(state):\n",
+ " \"\"\"\n",
+ " Determines whether the generation is grounded in the document and answers question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK HALLUCINATIONS---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ " generation = state[\"generation\"]\n",
+ "\n",
+ " score = hallucination_grader.invoke(\n",
+ " {\"documents\": documents, \"generation\": generation}\n",
+ " )\n",
+ " grade = score.binary_score\n",
+ "\n",
+ " # Check hallucination\n",
+ " if grade == \"yes\":\n",
+ " print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
+ " # Check question-answering\n",
+ " print(\"---GRADE GENERATION vs QUESTION---\")\n",
+ " score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
+ " grade = score.binary_score\n",
+ " if grade == \"yes\":\n",
+ " print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
+ " return \"useful\"\n",
+ " else:\n",
+ " print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
+ " return \"not useful\"\n",
+ " else:\n",
+ " pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
+ " return \"not supported\""
+ ]
},
{
"cell_type": "markdown",
@@ -262,7 +620,42 @@
"id": "0e09ca9f-e36d-4ef4-a0d5-79fdbada9fe0",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"transform_query\", transform_query) # transform_query\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"transform_query\": \"transform_query\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"transform_query\", \"retrieve\")\nworkflow.add_conditional_edges(\n \"generate\",\n grade_generation_v_documents_and_question,\n {\n \"not supported\": \"generate\",\n \"useful\": END,\n \"not useful\": \"transform_query\",\n },\n)\n\n# Compile\napp = workflow.compile()"]
+ "source": [
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "workflow = StateGraph(GraphState)\n",
+ "\n",
+ "# Define the nodes\n",
+ "workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
+ "workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
+ "workflow.add_node(\"generate\", generate) # generatae\n",
+ "workflow.add_node(\"transform_query\", transform_query) # transform_query\n",
+ "\n",
+ "# Build graph\n",
+ "workflow.add_edge(START, \"retrieve\")\n",
+ "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+ "workflow.add_conditional_edges(\n",
+ " \"grade_documents\",\n",
+ " decide_to_generate,\n",
+ " {\n",
+ " \"transform_query\": \"transform_query\",\n",
+ " \"generate\": \"generate\",\n",
+ " },\n",
+ ")\n",
+ "workflow.add_edge(\"transform_query\", \"retrieve\")\n",
+ "workflow.add_conditional_edges(\n",
+ " \"generate\",\n",
+ " grade_generation_v_documents_and_question,\n",
+ " {\n",
+ " \"not supported\": \"generate\",\n",
+ " \"useful\": END,\n",
+ " \"not useful\": \"transform_query\",\n",
+ " },\n",
+ ")\n",
+ "\n",
+ "# Compile\n",
+ "app = workflow.compile()"
+ ]
},
{
"cell_type": "code",
@@ -301,7 +694,22 @@
]
}
],
- "source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"Explain how the different types of agent memory work?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
+ "source": [
+ "from pprint import pprint\n",
+ "\n",
+ "# Run\n",
+ "inputs = {\"question\": \"Explain how the different types of agent memory work?\"}\n",
+ "for output in app.stream(inputs):\n",
+ " for key, value in output.items():\n",
+ " # Node\n",
+ " pprint(f\"Node '{key}':\")\n",
+ " # Optional: print full state at each node\n",
+ " # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+ " pprint(\"\\n---\\n\")\n",
+ "\n",
+ "# Final generation\n",
+ "pprint(value[\"generation\"])"
+ ]
},
{
"cell_type": "code",
@@ -341,7 +749,19 @@
]
}
],
- "source": ["inputs = {\"question\": \"Explain how chain of thought prompting works?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
+ "source": [
+ "inputs = {\"question\": \"Explain how chain of thought prompting works?\"}\n",
+ "for output in app.stream(inputs):\n",
+ " for key, value in output.items():\n",
+ " # Node\n",
+ " pprint(f\"Node '{key}':\")\n",
+ " # Optional: print full state at each node\n",
+ " # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+ " pprint(\"\\n---\\n\")\n",
+ "\n",
+ "# Final generation\n",
+ "pprint(value[\"generation\"])"
+ ]
},
{
"cell_type": "markdown",
@@ -354,14 +774,6 @@
"\n",
"* https://smith.langchain.com/public/1c6bf654-61b2-4fc5-9889-054b020c78aa/r"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "42369ab8-322d-434a-b5dd-2266e4cb2903",
- "metadata": {},
- "outputs": [],
- "source": [""]
}
],
"metadata": {
diff --git a/examples/rag/langgraph_self_rag_local.ipynb b/examples/rag/langgraph_self_rag_local.ipynb
index bc107d0b4..64b576ad4 100644
--- a/examples/rag/langgraph_self_rag_local.ipynb
+++ b/examples/rag/langgraph_self_rag_local.ipynb
@@ -59,7 +59,10 @@
"id": "d7f9cc6d-a70c-433a-b0ad-ea47c5a0717e",
"metadata": {},
"outputs": [],
- "source": ["%capture --no-stderr\n%pip install -U langchain-nomic langchain_community tiktoken langchainhub chromadb langchain langgraph nomic[local]"]
+ "source": [
+ "%capture --no-stderr\n",
+ "%pip install -U langchain-nomic langchain_community tiktoken langchainhub chromadb langchain langgraph nomic[local]"
+ ]
},
{
"cell_type": "markdown",
@@ -91,7 +94,10 @@
"id": "bedffc73-6b10-42c8-8768-2085c8ed3398",
"metadata": {},
"outputs": [],
- "source": ["# Ollama model name\nlocal_llm = \"mistral\""]
+ "source": [
+ "# Ollama model name\n",
+ "local_llm = \"mistral\""
+ ]
},
{
"cell_type": "markdown",
@@ -109,7 +115,13 @@
"id": "2208f342-8163-4af3-8dc0-aa70f5e06143",
"metadata": {},
"outputs": [],
- "source": ["import os\n\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"\""]
+ "source": [
+ "import os\n",
+ "\n",
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
+ "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+ "os.environ[\"LANGCHAIN_API_KEY\"] = \"\""
+ ]
},
{
"cell_type": "markdown",
@@ -127,7 +139,34 @@
"id": "c3bb9060-ad74-4470-9991-2ba167b6b8d8",
"metadata": {},
"outputs": [],
- "source": ["from langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_nomic.embeddings import NomicEmbeddings\n\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=250, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorDB\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n)\nretriever = vectorstore.as_retriever()"]
+ "source": [
+ "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+ "from langchain_community.document_loaders import WebBaseLoader\n",
+ "from langchain_community.vectorstores import Chroma\n",
+ "from langchain_nomic.embeddings import NomicEmbeddings\n",
+ "\n",
+ "urls = [\n",
+ " \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+ " \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+ "]\n",
+ "\n",
+ "docs = [WebBaseLoader(url).load() for url in urls]\n",
+ "docs_list = [item for sublist in docs for item in sublist]\n",
+ "\n",
+ "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+ " chunk_size=250, chunk_overlap=0\n",
+ ")\n",
+ "doc_splits = text_splitter.split_documents(docs_list)\n",
+ "\n",
+ "# Add to vectorDB\n",
+ "vectorstore = Chroma.from_documents(\n",
+ " documents=doc_splits,\n",
+ " collection_name=\"rag-chroma\",\n",
+ " embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n",
+ ")\n",
+ "retriever = vectorstore.as_retriever()"
+ ]
},
{
"cell_type": "markdown",
@@ -151,7 +190,33 @@
]
}
],
- "source": ["### Retrieval Grader\n\nfrom langchain.prompts import PromptTemplate\nfrom langchain_community.chat_models import ChatOllama\nfrom langchain_core.output_parsers import JsonOutputParser\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n Here is the retrieved document: \\n\\n {document} \\n\\n\n Here is the user question: {question} \\n\n If the document contains keywords related to the user question, grade it as relevant. \\n\n It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\"\"\",\n input_variables=[\"question\", \"document\"],\n)\n\nretrieval_grader = prompt | llm | JsonOutputParser()\nquestion = \"agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"]
+ "source": [
+ "### Retrieval Grader\n",
+ "\n",
+ "from langchain.prompts import PromptTemplate\n",
+ "from langchain_community.chat_models import ChatOllama\n",
+ "from langchain_core.output_parsers import JsonOutputParser\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+ "\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+ " Here is the retrieved document: \\n\\n {document} \\n\\n\n",
+ " Here is the user question: {question} \\n\n",
+ " If the document contains keywords related to the user question, grade it as relevant. \\n\n",
+ " It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
+ " Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n",
+ " Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\"\"\",\n",
+ " input_variables=[\"question\", \"document\"],\n",
+ ")\n",
+ "\n",
+ "retrieval_grader = prompt | llm | JsonOutputParser()\n",
+ "question = \"agent memory\"\n",
+ "docs = retriever.get_relevant_documents(question)\n",
+ "doc_txt = docs[1].page_content\n",
+ "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+ ]
},
{
"cell_type": "code",
@@ -167,7 +232,31 @@
]
}
],
- "source": ["### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOllama(model=local_llm, temperature=0)\n\n\n# Post-processing\ndef format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"]
+ "source": [
+ "### Generate\n",
+ "\n",
+ "from langchain import hub\n",
+ "from langchain_core.output_parsers import StrOutputParser\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, temperature=0)\n",
+ "\n",
+ "\n",
+ "# Post-processing\n",
+ "def format_docs(docs):\n",
+ " return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+ "\n",
+ "\n",
+ "# Chain\n",
+ "rag_chain = prompt | llm | StrOutputParser()\n",
+ "\n",
+ "# Run\n",
+ "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+ "print(generation)"
+ ]
},
{
"cell_type": "code",
@@ -186,7 +275,28 @@
"output_type": "execute_result"
}
],
- "source": ["### Hallucination Grader\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing whether an answer is grounded in / supported by a set of facts. \\n \n Here are the facts:\n \\n ------- \\n\n {documents} \n \\n ------- \\n\n Here is the answer: {generation}\n Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \\n\n Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n input_variables=[\"generation\", \"documents\"],\n)\n\nhallucination_grader = prompt | llm | JsonOutputParser()\nhallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"]
+ "source": [
+ "### Hallucination Grader\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are a grader assessing whether an answer is grounded in / supported by a set of facts. \\n \n",
+ " Here are the facts:\n",
+ " \\n ------- \\n\n",
+ " {documents} \n",
+ " \\n ------- \\n\n",
+ " Here is the answer: {generation}\n",
+ " Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \\n\n",
+ " Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
+ " input_variables=[\"generation\", \"documents\"],\n",
+ ")\n",
+ "\n",
+ "hallucination_grader = prompt | llm | JsonOutputParser()\n",
+ "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+ ]
},
{
"cell_type": "code",
@@ -205,7 +315,28 @@
"output_type": "execute_result"
}
],
- "source": ["### Answer Grader\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing whether an answer is useful to resolve a question. \\n \n Here is the answer:\n \\n ------- \\n\n {generation} \n \\n ------- \\n\n Here is the question: {question}\n Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \\n\n Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n input_variables=[\"generation\", \"question\"],\n)\n\nanswer_grader = prompt | llm | JsonOutputParser()\nanswer_grader.invoke({\"question\": question, \"generation\": generation})"]
+ "source": [
+ "### Answer Grader\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "prompt = PromptTemplate(\n",
+ " template=\"\"\"You are a grader assessing whether an answer is useful to resolve a question. \\n \n",
+ " Here is the answer:\n",
+ " \\n ------- \\n\n",
+ " {generation} \n",
+ " \\n ------- \\n\n",
+ " Here is the question: {question}\n",
+ " Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \\n\n",
+ " Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
+ " input_variables=[\"generation\", \"question\"],\n",
+ ")\n",
+ "\n",
+ "answer_grader = prompt | llm | JsonOutputParser()\n",
+ "answer_grader.invoke({\"question\": question, \"generation\": generation})"
+ ]
},
{
"cell_type": "code",
@@ -224,7 +355,23 @@
"output_type": "execute_result"
}
],
- "source": ["### Question Re-writer\n\n# LLM\nllm = ChatOllama(model=local_llm, temperature=0)\n\n# Prompt\nre_write_prompt = PromptTemplate(\n template=\"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n for vectorstore retrieval. Look at the initial and formulate an improved question. \\n\n Here is the initial question: \\n\\n {question}. Improved question with no preamble: \\n \"\"\",\n input_variables=[\"generation\", \"question\"],\n)\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nquestion_rewriter.invoke({\"question\": question})"]
+ "source": [
+ "### Question Re-writer\n",
+ "\n",
+ "# LLM\n",
+ "llm = ChatOllama(model=local_llm, temperature=0)\n",
+ "\n",
+ "# Prompt\n",
+ "re_write_prompt = PromptTemplate(\n",
+ " template=\"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+ " for vectorstore retrieval. Look at the initial and formulate an improved question. \\n\n",
+ " Here is the initial question: \\n\\n {question}. Improved question with no preamble: \\n \"\"\",\n",
+ " input_variables=[\"generation\", \"question\"],\n",
+ ")\n",
+ "\n",
+ "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+ "question_rewriter.invoke({\"question\": question})"
+ ]
},
{
"cell_type": "markdown",
@@ -244,7 +391,26 @@
"id": "90fb1dc6-c482-483a-8441-39965c401beb",
"metadata": {},
"outputs": [],
- "source": ["from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n documents: List[str]"]
+ "source": [
+ "from typing import List\n",
+ "\n",
+ "from typing_extensions import TypedDict\n",
+ "\n",
+ "\n",
+ "class GraphState(TypedDict):\n",
+ " \"\"\"\n",
+ " Represents the state of our graph.\n",
+ "\n",
+ " Attributes:\n",
+ " question: question\n",
+ " generation: LLM generation\n",
+ " documents: list of documents\n",
+ " \"\"\"\n",
+ "\n",
+ " question: str\n",
+ " generation: str\n",
+ " documents: List[str]"
+ ]
},
{
"cell_type": "code",
@@ -252,7 +418,167 @@
"id": "5324ea49-5745-47b5-a0a5-bf58c8babe46",
"metadata": {},
"outputs": [],
- "source": ["### Nodes\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.get_relevant_documents(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score[\"score\"]\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n continue\n return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}\n\n\n### Edges\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n filtered_documents = state[\"documents\"]\n\n if not filtered_documents:\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\"\n\n\ndef grade_generation_v_documents_and_question(state):\n \"\"\"\n Determines whether the generation is grounded in the document and answers question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Decision for next node to call\n \"\"\"\n\n print(\"---CHECK HALLUCINATIONS---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = state[\"generation\"]\n\n score = hallucination_grader.invoke(\n {\"documents\": documents, \"generation\": generation}\n )\n grade = score[\"score\"]\n\n # Check hallucination\n if grade == \"yes\":\n print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n # Check question-answering\n print(\"---GRADE GENERATION vs QUESTION---\")\n score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n grade = score[\"score\"]\n if grade == \"yes\":\n print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n return \"useful\"\n else:\n print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n return \"not useful\"\n else:\n print(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n return \"not supported\""]
+ "source": [
+ "### Nodes\n",
+ "\n",
+ "\n",
+ "def retrieve(state):\n",
+ " \"\"\"\n",
+ " Retrieve documents\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, documents, that contains retrieved documents\n",
+ " \"\"\"\n",
+ " print(\"---RETRIEVE---\")\n",
+ " question = state[\"question\"]\n",
+ "\n",
+ " # Retrieval\n",
+ " documents = retriever.get_relevant_documents(question)\n",
+ " return {\"documents\": documents, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def generate(state):\n",
+ " \"\"\"\n",
+ " Generate answer\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): New key added to state, generation, that contains LLM generation\n",
+ " \"\"\"\n",
+ " print(\"---GENERATE---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # RAG generation\n",
+ " generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+ " return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+ "\n",
+ "\n",
+ "def grade_documents(state):\n",
+ " \"\"\"\n",
+ " Determines whether the retrieved documents are relevant to the question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates documents key with only filtered relevant documents\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Score each doc\n",
+ " filtered_docs = []\n",
+ " for d in documents:\n",
+ " score = retrieval_grader.invoke(\n",
+ " {\"question\": question, \"document\": d.page_content}\n",
+ " )\n",
+ " grade = score[\"score\"]\n",
+ " if grade == \"yes\":\n",
+ " print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+ " filtered_docs.append(d)\n",
+ " else:\n",
+ " print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+ " continue\n",
+ " return {\"documents\": filtered_docs, \"question\": question}\n",
+ "\n",
+ "\n",
+ "def transform_query(state):\n",
+ " \"\"\"\n",
+ " Transform the query to produce a better question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " state (dict): Updates question key with a re-phrased question\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---TRANSFORM QUERY---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ "\n",
+ " # Re-write question\n",
+ " better_question = question_rewriter.invoke({\"question\": question})\n",
+ " return {\"documents\": documents, \"question\": better_question}\n",
+ "\n",
+ "\n",
+ "### Edges\n",
+ "\n",
+ "\n",
+ "def decide_to_generate(state):\n",
+ " \"\"\"\n",
+ " Determines whether to generate an answer, or re-generate a question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Binary decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+ " state[\"question\"]\n",
+ " filtered_documents = state[\"documents\"]\n",
+ "\n",
+ " if not filtered_documents:\n",
+ " # All documents have been filtered check_relevance\n",
+ " # We will re-generate a new query\n",
+ " print(\n",
+ " \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+ " )\n",
+ " return \"transform_query\"\n",
+ " else:\n",
+ " # We have relevant documents, so generate answer\n",
+ " print(\"---DECISION: GENERATE---\")\n",
+ " return \"generate\"\n",
+ "\n",
+ "\n",
+ "def grade_generation_v_documents_and_question(state):\n",
+ " \"\"\"\n",
+ " Determines whether the generation is grounded in the document and answers question.\n",
+ "\n",
+ " Args:\n",
+ " state (dict): The current graph state\n",
+ "\n",
+ " Returns:\n",
+ " str: Decision for next node to call\n",
+ " \"\"\"\n",
+ "\n",
+ " print(\"---CHECK HALLUCINATIONS---\")\n",
+ " question = state[\"question\"]\n",
+ " documents = state[\"documents\"]\n",
+ " generation = state[\"generation\"]\n",
+ "\n",
+ " score = hallucination_grader.invoke(\n",
+ " {\"documents\": documents, \"generation\": generation}\n",
+ " )\n",
+ " grade = score[\"score\"]\n",
+ "\n",
+ " # Check hallucination\n",
+ " if grade == \"yes\":\n",
+ " print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
+ " # Check question-answering\n",
+ " print(\"---GRADE GENERATION vs QUESTION---\")\n",
+ " score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
+ " grade = score[\"score\"]\n",
+ " if grade == \"yes\":\n",
+ " print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
+ " return \"useful\"\n",
+ " else:\n",
+ " print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
+ " return \"not useful\"\n",
+ " else:\n",
+ " print(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
+ " return \"not supported\""
+ ]
},
{
"cell_type": "markdown",
@@ -270,7 +596,42 @@
"id": "5605dee4-b2df-46ae-a640-cc2ed90c21a6",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"transform_query\", transform_query) # transform_query\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"transform_query\": \"transform_query\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"transform_query\", \"retrieve\")\nworkflow.add_conditional_edges(\n \"generate\",\n grade_generation_v_documents_and_question,\n {\n \"not supported\": \"generate\",\n \"useful\": END,\n \"not useful\": \"transform_query\",\n },\n)\n\n# Compile\napp = workflow.compile()"]
+ "source": [
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "workflow = StateGraph(GraphState)\n",
+ "\n",
+ "# Define the nodes\n",
+ "workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
+ "workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
+ "workflow.add_node(\"generate\", generate) # generatae\n",
+ "workflow.add_node(\"transform_query\", transform_query) # transform_query\n",
+ "\n",
+ "# Build graph\n",
+ "workflow.add_edge(START, \"retrieve\")\n",
+ "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+ "workflow.add_conditional_edges(\n",
+ " \"grade_documents\",\n",
+ " decide_to_generate,\n",
+ " {\n",
+ " \"transform_query\": \"transform_query\",\n",
+ " \"generate\": \"generate\",\n",
+ " },\n",
+ ")\n",
+ "workflow.add_edge(\"transform_query\", \"retrieve\")\n",
+ "workflow.add_conditional_edges(\n",
+ " \"generate\",\n",
+ " grade_generation_v_documents_and_question,\n",
+ " {\n",
+ " \"not supported\": \"generate\",\n",
+ " \"useful\": END,\n",
+ " \"not useful\": \"transform_query\",\n",
+ " },\n",
+ ")\n",
+ "\n",
+ "# Compile\n",
+ "app = workflow.compile()"
+ ]
},
{
"cell_type": "markdown",
@@ -324,7 +685,22 @@
]
}
],
- "source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"Explain how the different types of agent memory work?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
+ "source": [
+ "from pprint import pprint\n",
+ "\n",
+ "# Run\n",
+ "inputs = {\"question\": \"Explain how the different types of agent memory work?\"}\n",
+ "for output in app.stream(inputs):\n",
+ " for key, value in output.items():\n",
+ " # Node\n",
+ " pprint(f\"Node '{key}':\")\n",
+ " # Optional: print full state at each node\n",
+ " # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+ " pprint(\"\\n---\\n\")\n",
+ "\n",
+ "# Final generation\n",
+ "pprint(value[\"generation\"])"
+ ]
},
{
"cell_type": "markdown",
@@ -335,14 +711,6 @@
"\n",
"https://smith.langchain.com/public/4163a342-5260-4852-8602-bda3f95177e7/r"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "953143c2-2f2a-4361-a36b-87db7cf21d63",
- "metadata": {},
- "outputs": [],
- "source": [""]
}
],
"metadata": {
diff --git a/examples/streaming-tokens-without-langchain.ipynb b/examples/streaming-tokens-without-langchain.ipynb
index 40ff751e0..1284d0662 100644
--- a/examples/streaming-tokens-without-langchain.ipynb
+++ b/examples/streaming-tokens-without-langchain.ipynb
@@ -30,7 +30,10 @@
"id": "47f79af8-58d8-4a48-8d9a-88823d88701f",
"metadata": {},
"outputs": [],
- "source": ["%%capture --no-stderr\n%pip install -U langgraph openai"]
+ "source": [
+ "%%capture --no-stderr\n",
+ "%pip install -U langgraph openai"
+ ]
},
{
"cell_type": "code",
@@ -46,7 +49,18 @@
]
}
],
- "source": ["import getpass\nimport os\n\n\ndef _set_env(var: str):\n if not os.environ.get(var):\n os.environ[var] = getpass.getpass(f\"{var}: \")\n\n\n_set_env(\"OPENAI_API_KEY\")"]
+ "source": [
+ "import getpass\n",
+ "import os\n",
+ "\n",
+ "\n",
+ "def _set_env(var: str):\n",
+ " if not os.environ.get(var):\n",
+ " os.environ[var] = getpass.getpass(f\"{var}: \")\n",
+ "\n",
+ "\n",
+ "_set_env(\"OPENAI_API_KEY\")"
+ ]
},
{
"cell_type": "markdown",
@@ -70,7 +84,94 @@
"id": "d59234f9-173e-469d-a725-c13e0979663e",
"metadata": {},
"outputs": [],
- "source": ["from openai import AsyncOpenAI\nfrom langchain_core.language_models.chat_models import ChatGenerationChunk\nfrom langchain_core.messages import AIMessageChunk\nfrom langchain_core.runnables.config import (\n ensure_config,\n get_callback_manager_for_config,\n)\n\nopenai_client = AsyncOpenAI()\n# define tool schema for openai tool calling\n\ntool = {\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_items\",\n \"description\": \"Use this tool to look up which items are in the given place.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\"place\": {\"type\": \"string\"}},\n \"required\": [\"place\"],\n },\n },\n}\n\n\nasync def call_model(state, config=None):\n config = ensure_config(config | {\"tags\": [\"agent_llm\"]})\n callback_manager = get_callback_manager_for_config(config)\n messages = state[\"messages\"]\n\n llm_run_manager = callback_manager.on_chat_model_start({}, [messages])[0]\n response = await openai_client.chat.completions.create(\n messages=messages, model=\"gpt-3.5-turbo\", tools=[tool], stream=True\n )\n\n response_content = \"\"\n role = None\n\n tool_call_id = None\n tool_call_function_name = None\n tool_call_function_arguments = \"\"\n async for chunk in response:\n delta = chunk.choices[0].delta\n if delta.role is not None:\n role = delta.role\n\n if delta.content:\n response_content += delta.content\n llm_run_manager.on_llm_new_token(delta.content)\n\n if delta.tool_calls:\n # note: for simplicity we're only handling a single tool call here\n if delta.tool_calls[0].function.name is not None:\n tool_call_function_name = delta.tool_calls[0].function.name\n tool_call_id = delta.tool_calls[0].id\n\n # note: we're wrapping the tools calls in ChatGenerationChunk so that the events from .astream_events in the graph can render tool calls correctly\n tool_call_chunk = ChatGenerationChunk(\n message=AIMessageChunk(\n content=\"\",\n additional_kwargs={\"tool_calls\": [delta.tool_calls[0].dict()]},\n )\n )\n llm_run_manager.on_llm_new_token(\"\", chunk=tool_call_chunk)\n tool_call_function_arguments += delta.tool_calls[0].function.arguments\n\n if tool_call_function_name is not None:\n tool_calls = [\n {\n \"id\": tool_call_id,\n \"function\": {\n \"name\": tool_call_function_name,\n \"arguments\": tool_call_function_arguments,\n },\n \"type\": \"function\",\n }\n ]\n else:\n tool_calls = None\n\n response_message = {\n \"role\": role,\n \"content\": response_content,\n \"tool_calls\": tool_calls,\n }\n return {\"messages\": [response_message]}"]
+ "source": [
+ "from openai import AsyncOpenAI\n",
+ "from langchain_core.language_models.chat_models import ChatGenerationChunk\n",
+ "from langchain_core.messages import AIMessageChunk\n",
+ "from langchain_core.runnables.config import (\n",
+ " ensure_config,\n",
+ " get_callback_manager_for_config,\n",
+ ")\n",
+ "\n",
+ "openai_client = AsyncOpenAI()\n",
+ "# define tool schema for openai tool calling\n",
+ "\n",
+ "tool = {\n",
+ " \"type\": \"function\",\n",
+ " \"function\": {\n",
+ " \"name\": \"get_items\",\n",
+ " \"description\": \"Use this tool to look up which items are in the given place.\",\n",
+ " \"parameters\": {\n",
+ " \"type\": \"object\",\n",
+ " \"properties\": {\"place\": {\"type\": \"string\"}},\n",
+ " \"required\": [\"place\"],\n",
+ " },\n",
+ " },\n",
+ "}\n",
+ "\n",
+ "\n",
+ "async def call_model(state, config=None):\n",
+ " config = ensure_config(config | {\"tags\": [\"agent_llm\"]})\n",
+ " callback_manager = get_callback_manager_for_config(config)\n",
+ " messages = state[\"messages\"]\n",
+ "\n",
+ " llm_run_manager = callback_manager.on_chat_model_start({}, [messages])[0]\n",
+ " response = await openai_client.chat.completions.create(\n",
+ " messages=messages, model=\"gpt-3.5-turbo\", tools=[tool], stream=True\n",
+ " )\n",
+ "\n",
+ " response_content = \"\"\n",
+ " role = None\n",
+ "\n",
+ " tool_call_id = None\n",
+ " tool_call_function_name = None\n",
+ " tool_call_function_arguments = \"\"\n",
+ " async for chunk in response:\n",
+ " delta = chunk.choices[0].delta\n",
+ " if delta.role is not None:\n",
+ " role = delta.role\n",
+ "\n",
+ " if delta.content:\n",
+ " response_content += delta.content\n",
+ " llm_run_manager.on_llm_new_token(delta.content)\n",
+ "\n",
+ " if delta.tool_calls:\n",
+ " # note: for simplicity we're only handling a single tool call here\n",
+ " if delta.tool_calls[0].function.name is not None:\n",
+ " tool_call_function_name = delta.tool_calls[0].function.name\n",
+ " tool_call_id = delta.tool_calls[0].id\n",
+ "\n",
+ " # note: we're wrapping the tools calls in ChatGenerationChunk so that the events from .astream_events in the graph can render tool calls correctly\n",
+ " tool_call_chunk = ChatGenerationChunk(\n",
+ " message=AIMessageChunk(\n",
+ " content=\"\",\n",
+ " additional_kwargs={\"tool_calls\": [delta.tool_calls[0].dict()]},\n",
+ " )\n",
+ " )\n",
+ " llm_run_manager.on_llm_new_token(\"\", chunk=tool_call_chunk)\n",
+ " tool_call_function_arguments += delta.tool_calls[0].function.arguments\n",
+ "\n",
+ " if tool_call_function_name is not None:\n",
+ " tool_calls = [\n",
+ " {\n",
+ " \"id\": tool_call_id,\n",
+ " \"function\": {\n",
+ " \"name\": tool_call_function_name,\n",
+ " \"arguments\": tool_call_function_arguments,\n",
+ " },\n",
+ " \"type\": \"function\",\n",
+ " }\n",
+ " ]\n",
+ " else:\n",
+ " tool_calls = None\n",
+ "\n",
+ " response_message = {\n",
+ " \"role\": role,\n",
+ " \"content\": response_content,\n",
+ " \"tool_calls\": tool_calls,\n",
+ " }\n",
+ " return {\"messages\": [response_message]}"
+ ]
},
{
"cell_type": "markdown",
@@ -86,7 +187,41 @@
"id": "b756ea32",
"metadata": {},
"outputs": [],
- "source": ["import json\n\n\nasync def get_items(place: str) -> str:\n \"\"\"Use this tool to look up which items are in the given place.\"\"\"\n if \"bed\" in place: # For under the bed\n return \"socks, shoes and dust bunnies\"\n if \"shelf\" in place: # For 'shelf'\n return \"books, penciles and pictures\"\n else: # if the agent decides to ask about a different place\n return \"cat snacks\"\n\n\n# define mapping to look up functions when running tools\nfunction_name_to_function = {\"get_items\": get_items}\n\n\nasync def call_tools(state):\n messages = state[\"messages\"]\n\n tool_call = messages[-1][\"tool_calls\"][0]\n function_name = tool_call[\"function\"][\"name\"]\n function_arguments = tool_call[\"function\"][\"arguments\"]\n arguments = json.loads(function_arguments)\n\n function_response = await function_name_to_function[function_name](**arguments)\n tool_message = {\n \"tool_call_id\": tool_call[\"id\"],\n \"role\": \"tool\",\n \"name\": function_name,\n \"content\": function_response,\n }\n return {\"messages\": [tool_message]}"]
+ "source": [
+ "import json\n",
+ "\n",
+ "\n",
+ "async def get_items(place: str) -> str:\n",
+ " \"\"\"Use this tool to look up which items are in the given place.\"\"\"\n",
+ " if \"bed\" in place: # For under the bed\n",
+ " return \"socks, shoes and dust bunnies\"\n",
+ " if \"shelf\" in place: # For 'shelf'\n",
+ " return \"books, penciles and pictures\"\n",
+ " else: # if the agent decides to ask about a different place\n",
+ " return \"cat snacks\"\n",
+ "\n",
+ "\n",
+ "# define mapping to look up functions when running tools\n",
+ "function_name_to_function = {\"get_items\": get_items}\n",
+ "\n",
+ "\n",
+ "async def call_tools(state):\n",
+ " messages = state[\"messages\"]\n",
+ "\n",
+ " tool_call = messages[-1][\"tool_calls\"][0]\n",
+ " function_name = tool_call[\"function\"][\"name\"]\n",
+ " function_arguments = tool_call[\"function\"][\"arguments\"]\n",
+ " arguments = json.loads(function_arguments)\n",
+ "\n",
+ " function_response = await function_name_to_function[function_name](**arguments)\n",
+ " tool_message = {\n",
+ " \"tool_call_id\": tool_call[\"id\"],\n",
+ " \"role\": \"tool\",\n",
+ " \"name\": function_name,\n",
+ " \"content\": function_response,\n",
+ " }\n",
+ " return {\"messages\": [tool_message]}"
+ ]
},
{
"cell_type": "markdown",
@@ -102,7 +237,33 @@
"id": "228260be-1f9a-4195-80e0-9604f8a5dba6",
"metadata": {},
"outputs": [],
- "source": ["import operator\nfrom typing import Annotated, TypedDict, Literal\n\nfrom langgraph.graph import StateGraph, END, START\n\n\nclass State(TypedDict):\n messages: Annotated[list, operator.add]\n\n\ndef should_continue(state) -> Literal[\"tools\", END]:\n messages = state[\"messages\"]\n last_message = messages[-1]\n if last_message[\"tool_calls\"]:\n return \"tools\"\n return END\n\n\nworkflow = StateGraph(State)\nworkflow.add_edge(START, \"model\")\nworkflow.add_node(\"model\", call_model) # i.e. our \"agent\"\nworkflow.add_node(\"tools\", call_tools)\nworkflow.add_conditional_edges(\"model\", should_continue)\nworkflow.add_edge(\"tools\", \"model\")\ngraph = workflow.compile()"]
+ "source": [
+ "import operator\n",
+ "from typing import Annotated, TypedDict, Literal\n",
+ "\n",
+ "from langgraph.graph import StateGraph, END, START\n",
+ "\n",
+ "\n",
+ "class State(TypedDict):\n",
+ " messages: Annotated[list, operator.add]\n",
+ "\n",
+ "\n",
+ "def should_continue(state) -> Literal[\"tools\", END]:\n",
+ " messages = state[\"messages\"]\n",
+ " last_message = messages[-1]\n",
+ " if last_message[\"tool_calls\"]:\n",
+ " return \"tools\"\n",
+ " return END\n",
+ "\n",
+ "\n",
+ "workflow = StateGraph(State)\n",
+ "workflow.add_edge(START, \"model\")\n",
+ "workflow.add_node(\"model\", call_model) # i.e. our \"agent\"\n",
+ "workflow.add_node(\"tools\", call_tools)\n",
+ "workflow.add_conditional_edges(\"model\", should_continue)\n",
+ "workflow.add_edge(\"tools\", \"model\")\n",
+ "graph = workflow.compile()"
+ ]
},
{
"cell_type": "markdown",
@@ -167,15 +328,14 @@
]
}
],
- "source": ["async for event in graph.astream_events(\n {\"messages\": [{\"role\": \"user\", \"content\": \"what's in the bedroom\"}]}, version=\"v2\"\n):\n tags = event.get(\"tags\", [])\n if event[\"event\"] == \"on_chat_model_stream\" and \"agent_llm\" in tags:\n print(\"LLM token\", event[\"data\"][\"chunk\"].dict())"]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "adb0f7bc-6e51-478e-bd32-8f72df072d6c",
- "metadata": {},
- "outputs": [],
- "source": [""]
+ "source": [
+ "async for event in graph.astream_events(\n",
+ " {\"messages\": [{\"role\": \"user\", \"content\": \"what's in the bedroom\"}]}, version=\"v2\"\n",
+ "):\n",
+ " tags = event.get(\"tags\", [])\n",
+ " if event[\"event\"] == \"on_chat_model_stream\" and \"agent_llm\" in tags:\n",
+ " print(\"LLM token\", event[\"data\"][\"chunk\"].dict())"
+ ]
}
],
"metadata": {
diff --git a/examples/usaco/usaco.ipynb b/examples/usaco/usaco.ipynb
index 0a4d4e445..16f698fce 100644
--- a/examples/usaco/usaco.ipynb
+++ b/examples/usaco/usaco.ipynb
@@ -43,7 +43,10 @@
"id": "c686827a-8078-4fd4-af7a-638ca1362796",
"metadata": {},
"outputs": [],
- "source": ["%%capture --no-stderr\n%pip install -U langgraph langsmith langchain_anthropic datasets langchain langchainhub"]
+ "source": [
+ "%%capture --no-stderr\n",
+ "%pip install -U langgraph langsmith langchain_anthropic datasets langchain langchainhub"
+ ]
},
{
"cell_type": "code",
@@ -51,7 +54,21 @@
"id": "e2e542bb-a99e-44d3-8ebb-6a952dcbf2bf",
"metadata": {},
"outputs": [],
- "source": ["import getpass\nimport os\n\n\ndef _get_env(var: str):\n if not os.environ.get(var):\n os.environ[var] = getpass.getpass(f\"{var}: \")\n\n\n_get_env(\"ANTHROPIC_API_KEY\")\n# Recommended\n_get_env(\"LANGCHAIN_API_KEY\")\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\""]
+ "source": [
+ "import getpass\n",
+ "import os\n",
+ "\n",
+ "\n",
+ "def _get_env(var: str):\n",
+ " if not os.environ.get(var):\n",
+ " os.environ[var] = getpass.getpass(f\"{var}: \")\n",
+ "\n",
+ "\n",
+ "_get_env(\"ANTHROPIC_API_KEY\")\n",
+ "# Recommended\n",
+ "_get_env(\"LANGCHAIN_API_KEY\")\n",
+ "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\""
+ ]
},
{
"cell_type": "markdown",
@@ -69,7 +86,28 @@
"id": "f7a0c7bd-512d-4e5b-ab43-1bc3b8c97fd4",
"metadata": {},
"outputs": [],
- "source": ["import os\nimport zipfile\n\nimport datasets\nimport requests\n\nusaco_url = \"https://storage.googleapis.com/benchmarks-artifacts/usaco/usaco_sampled_with_tests.zip\"\nzip_path = \"usaco.zip\"\nextract_path = \"usaco_datasets\"\n\nresponse = requests.get(usaco_url)\nwith open(zip_path, \"wb\") as file:\n file.write(response.content)\n\nwith zipfile.ZipFile(zip_path, \"r\") as zip_ref:\n zip_ref.extractall(extract_path)\n\nos.remove(zip_path)\n\nds = datasets.load_from_disk(os.path.join(extract_path, \"usaco_v3_sampled_with_tests\"))"]
+ "source": [
+ "import os\n",
+ "import zipfile\n",
+ "\n",
+ "import datasets\n",
+ "import requests\n",
+ "\n",
+ "usaco_url = \"https://storage.googleapis.com/benchmarks-artifacts/usaco/usaco_sampled_with_tests.zip\"\n",
+ "zip_path = \"usaco.zip\"\n",
+ "extract_path = \"usaco_datasets\"\n",
+ "\n",
+ "response = requests.get(usaco_url)\n",
+ "with open(zip_path, \"wb\") as file:\n",
+ " file.write(response.content)\n",
+ "\n",
+ "with zipfile.ZipFile(zip_path, \"r\") as zip_ref:\n",
+ " zip_ref.extractall(extract_path)\n",
+ "\n",
+ "os.remove(zip_path)\n",
+ "\n",
+ "ds = datasets.load_from_disk(os.path.join(extract_path, \"usaco_v3_sampled_with_tests\"))"
+ ]
},
{
"cell_type": "markdown",
@@ -88,7 +126,72 @@
"id": "54f9d037-121e-412f-857a-3e0ccc73892e",
"metadata": {},
"outputs": [],
- "source": ["import multiprocessing\nimport queue\nimport subprocess\nimport sys\nimport time\nimport traceback\n\nmultiprocessing.set_start_method(\"fork\", force=True)\n# WARNING\n# This program exists to execute untrusted model-generated code. Although\n# it is highly unlikely that model-generated code will do something overtly\n# malicious in response to this test suite, model-generated code may act\n# destructively due to a lack of model capability or alignment.\n# Users are strongly encouraged to sandbox this evaluation suite so that it\n# does not perform destructive actions on their host or network.\n# Proceed at your own risk:\n\n\ndef exec_program(q, program, input_data, expected_output, timeout):\n try:\n start_time = time.time()\n process = subprocess.Popen(\n [sys.executable, \"-c\", program],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True,\n )\n stdout, stderr = process.communicate(input=input_data, timeout=timeout)\n if time.time() - start_time > timeout:\n raise TimeoutError(\"Execution timed out.\")\n if process.returncode != 0:\n q.put(f\"failed: {stderr}\")\n else:\n if stdout.strip() == expected_output.strip():\n q.put(\"passed\")\n else:\n q.put(f\"wrong answer. Expected '{expected_output}', got '{stdout}'\")\n except subprocess.TimeoutExpired:\n process.kill()\n q.put(\"timed out\")\n except Exception:\n q.put(f\"failed: {traceback.format_exc()}\")\n\n\ndef check_correctness(\n program: str, input_data: str, expected_output: str, timeout: float\n) -> str:\n q = multiprocessing.Queue()\n process = multiprocessing.Process(\n target=exec_program, args=(q, program, input_data, expected_output, timeout)\n )\n process.start()\n process.join(timeout=timeout + 1)\n if process.is_alive():\n process.terminate()\n process.join()\n result = \"timed out\"\n else:\n try:\n result = q.get_nowait()\n except queue.Empty:\n result = \"no result returned\"\n return result"]
+ "source": [
+ "import multiprocessing\n",
+ "import queue\n",
+ "import subprocess\n",
+ "import sys\n",
+ "import time\n",
+ "import traceback\n",
+ "\n",
+ "multiprocessing.set_start_method(\"fork\", force=True)\n",
+ "# WARNING\n",
+ "# This program exists to execute untrusted model-generated code. Although\n",
+ "# it is highly unlikely that model-generated code will do something overtly\n",
+ "# malicious in response to this test suite, model-generated code may act\n",
+ "# destructively due to a lack of model capability or alignment.\n",
+ "# Users are strongly encouraged to sandbox this evaluation suite so that it\n",
+ "# does not perform destructive actions on their host or network.\n",
+ "# Proceed at your own risk:\n",
+ "\n",
+ "\n",
+ "def exec_program(q, program, input_data, expected_output, timeout):\n",
+ " try:\n",
+ " start_time = time.time()\n",
+ " process = subprocess.Popen(\n",
+ " [sys.executable, \"-c\", program],\n",
+ " stdin=subprocess.PIPE,\n",
+ " stdout=subprocess.PIPE,\n",
+ " stderr=subprocess.PIPE,\n",
+ " text=True,\n",
+ " )\n",
+ " stdout, stderr = process.communicate(input=input_data, timeout=timeout)\n",
+ " if time.time() - start_time > timeout:\n",
+ " raise TimeoutError(\"Execution timed out.\")\n",
+ " if process.returncode != 0:\n",
+ " q.put(f\"failed: {stderr}\")\n",
+ " else:\n",
+ " if stdout.strip() == expected_output.strip():\n",
+ " q.put(\"passed\")\n",
+ " else:\n",
+ " q.put(f\"wrong answer. Expected '{expected_output}', got '{stdout}'\")\n",
+ " except subprocess.TimeoutExpired:\n",
+ " process.kill()\n",
+ " q.put(\"timed out\")\n",
+ " except Exception:\n",
+ " q.put(f\"failed: {traceback.format_exc()}\")\n",
+ "\n",
+ "\n",
+ "def check_correctness(\n",
+ " program: str, input_data: str, expected_output: str, timeout: float\n",
+ ") -> str:\n",
+ " q = multiprocessing.Queue()\n",
+ " process = multiprocessing.Process(\n",
+ " target=exec_program, args=(q, program, input_data, expected_output, timeout)\n",
+ " )\n",
+ " process.start()\n",
+ " process.join(timeout=timeout + 1)\n",
+ " if process.is_alive():\n",
+ " process.terminate()\n",
+ " process.join()\n",
+ " result = \"timed out\"\n",
+ " else:\n",
+ " try:\n",
+ " result = q.get_nowait()\n",
+ " except queue.Empty:\n",
+ " result = \"no result returned\"\n",
+ " return result"
+ ]
},
{
"cell_type": "markdown",
@@ -114,7 +217,17 @@
]
}
],
- "source": ["program_code = \"print('hello, world!')\"\ninput_data = \"\"\nexpected_output = \"hello, world!\"\ntimeout = 2\n\ntest_result = check_correctness(program_code, input_data, expected_output, timeout)\nprint(\"Example 1: \", test_result)\ntest_result = check_correctness(\"print('goodbye')\", input_data, \"hi there\", timeout)\nprint(\"Example 2: \", test_result)"]
+ "source": [
+ "program_code = \"print('hello, world!')\"\n",
+ "input_data = \"\"\n",
+ "expected_output = \"hello, world!\"\n",
+ "timeout = 2\n",
+ "\n",
+ "test_result = check_correctness(program_code, input_data, expected_output, timeout)\n",
+ "print(\"Example 1: \", test_result)\n",
+ "test_result = check_correctness(\"print('goodbye')\", input_data, \"hi there\", timeout)\n",
+ "print(\"Example 2: \", test_result)"
+ ]
},
{
"cell_type": "markdown",
@@ -152,7 +265,27 @@
"id": "f43d68d9-10be-4544-879a-88a33db18bea",
"metadata": {},
"outputs": [],
- "source": ["from typing import Annotated\n\nfrom typing_extensions import TypedDict\n\nfrom langgraph.graph.message import AnyMessage, add_messages\n\n\nclass TestCase(TypedDict):\n inputs: str\n outputs: str\n\n\nclass State(TypedDict):\n # Append-only chat memory so the agent can try to recover from initial mistakes.\n messages: Annotated[list[AnyMessage], add_messages]\n # From the dataset. These are used for testing.\n test_cases: list[TestCase]\n runtime_limit: int\n status: str"]
+ "source": [
+ "from typing import Annotated\n",
+ "\n",
+ "from typing_extensions import TypedDict\n",
+ "\n",
+ "from langgraph.graph.message import AnyMessage, add_messages\n",
+ "\n",
+ "\n",
+ "class TestCase(TypedDict):\n",
+ " inputs: str\n",
+ " outputs: str\n",
+ "\n",
+ "\n",
+ "class State(TypedDict):\n",
+ " # Append-only chat memory so the agent can try to recover from initial mistakes.\n",
+ " messages: Annotated[list[AnyMessage], add_messages]\n",
+ " # From the dataset. These are used for testing.\n",
+ " test_cases: list[TestCase]\n",
+ " runtime_limit: int\n",
+ " status: str"
+ ]
},
{
"cell_type": "markdown",
@@ -168,7 +301,18 @@
"id": "6d56776f-993b-4ca7-89ef-21dec01dc9d3",
"metadata": {},
"outputs": [],
- "source": ["input_states = [\n {\n \"messages\": [(\"user\", row[\"description\"])],\n \"test_cases\": row[\"test_cases\"],\n \"runtime_limit\": row[\"runtime_limit\"],\n \"status\": \"in_progress\",\n \"problem_level\": row[\"problem_level\"],\n }\n for row in ds\n]"]
+ "source": [
+ "input_states = [\n",
+ " {\n",
+ " \"messages\": [(\"user\", row[\"description\"])],\n",
+ " \"test_cases\": row[\"test_cases\"],\n",
+ " \"runtime_limit\": row[\"runtime_limit\"],\n",
+ " \"status\": \"in_progress\",\n",
+ " \"problem_level\": row[\"problem_level\"],\n",
+ " }\n",
+ " for row in ds\n",
+ "]"
+ ]
},
{
"cell_type": "markdown",
@@ -186,7 +330,28 @@
"id": "7b9e7742-16a3-4ad2-bc63-5f9cd4fd734b",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.language_models import BaseChatModel\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\n\n\nclass writePython(BaseModel):\n \"\"\"Write python code that resolves the problem.\"\"\"\n\n reasoning: str = Field(..., description=\"Conceptual solution.\")\n pseudocode: str = Field(..., description=\"Detailed English pseudocode.\")\n code: str = Field(..., description=\"Valid Python 3 solution to the problem\")\n\n\nclass Solver:\n def __init__(self, llm: BaseChatModel, prompt: ChatPromptTemplate):\n self.runnable = prompt | llm.bind_tools([writePython])\n\n def __call__(self, state: State) -> dict:\n # Our agent only can see the \"messages\" and will ignore the test info\n return {\"messages\": [self.runnable.invoke({\"messages\": state[\"messages\"]})]}"]
+ "source": [
+ "from langchain_core.language_models import BaseChatModel\n",
+ "from langchain_core.prompts import ChatPromptTemplate\n",
+ "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+ "\n",
+ "\n",
+ "class writePython(BaseModel):\n",
+ " \"\"\"Write python code that resolves the problem.\"\"\"\n",
+ "\n",
+ " reasoning: str = Field(..., description=\"Conceptual solution.\")\n",
+ " pseudocode: str = Field(..., description=\"Detailed English pseudocode.\")\n",
+ " code: str = Field(..., description=\"Valid Python 3 solution to the problem\")\n",
+ "\n",
+ "\n",
+ "class Solver:\n",
+ " def __init__(self, llm: BaseChatModel, prompt: ChatPromptTemplate):\n",
+ " self.runnable = prompt | llm.bind_tools([writePython])\n",
+ "\n",
+ " def __call__(self, state: State) -> dict:\n",
+ " # Our agent only can see the \"messages\" and will ignore the test info\n",
+ " return {\"messages\": [self.runnable.invoke({\"messages\": state[\"messages\"]})]}"
+ ]
},
{
"cell_type": "markdown",
@@ -231,7 +396,22 @@
]
}
],
- "source": ["from langchain import hub\nfrom langchain_anthropic import ChatAnthropic\n\n# For this section, we are testing zero-shot performance and won't have\n# any examples. Partial them out to pre-fill the template.\nprompt = hub.pull(\"wfh/usaco-draft-solver\").partial(examples=\"\")\nprint(\"*\" * 35 + \"Prompt\" + \"*\" * 35)\nprompt.pretty_print()\n\n# Use Haiku if you want to save $$ while (almost) never correctly answering the question\n# llm = ChatAnthropic(model=\"claude-3-haiku-20240307\")\nllm = ChatAnthropic(model=\"claude-3-opus-20240229\")\n\nsolver = Solver(llm, prompt)"]
+ "source": [
+ "from langchain import hub\n",
+ "from langchain_anthropic import ChatAnthropic\n",
+ "\n",
+ "# For this section, we are testing zero-shot performance and won't have\n",
+ "# any examples. Partial them out to pre-fill the template.\n",
+ "prompt = hub.pull(\"wfh/usaco-draft-solver\").partial(examples=\"\")\n",
+ "print(\"*\" * 35 + \"Prompt\" + \"*\" * 35)\n",
+ "prompt.pretty_print()\n",
+ "\n",
+ "# Use Haiku if you want to save $$ while (almost) never correctly answering the question\n",
+ "# llm = ChatAnthropic(model=\"claude-3-haiku-20240307\")\n",
+ "llm = ChatAnthropic(model=\"claude-3-opus-20240229\")\n",
+ "\n",
+ "solver = Solver(llm, prompt)"
+ ]
},
{
"cell_type": "code",
@@ -250,7 +430,25 @@
]
}
],
- "source": ["print(\"*\" * 34 + \" Example \" + \"*\" * 34)\nresult = solver(\n {\n \"messages\": [\n (\n \"user\",\n \"How do I get a perfectly random sample from an infinite stream\",\n )\n ]\n }\n)\nresult[\"messages\"][0].pretty_print()\n# Could expand to include (1)\n# 1. Restate the problem in plain English\n# 2. Closely following the explanation, restate and explain the solution in plain English\n# 3. Write a pseudocode solution\n# 4. Output the final Python solution with your solution steps in comments."]
+ "source": [
+ "print(\"*\" * 34 + \" Example \" + \"*\" * 34)\n",
+ "result = solver(\n",
+ " {\n",
+ " \"messages\": [\n",
+ " (\n",
+ " \"user\",\n",
+ " \"How do I get a perfectly random sample from an infinite stream\",\n",
+ " )\n",
+ " ]\n",
+ " }\n",
+ ")\n",
+ "result[\"messages\"][0].pretty_print()\n",
+ "# Could expand to include (1)\n",
+ "# 1. Restate the problem in plain English\n",
+ "# 2. Closely following the explanation, restate and explain the solution in plain English\n",
+ "# 3. Write a pseudocode solution\n",
+ "# 4. Output the final Python solution with your solution steps in comments."
+ ]
},
{
"cell_type": "markdown",
@@ -269,7 +467,57 @@
"id": "1785015b-24f8-415f-b950-e229b5137887",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.messages import AIMessage, HumanMessage, ToolMessage\n\n\n# This is the node we will add to the graph.\n# Most tool-calling APIs require that the `ToolMessage` contain the ID\n# of the\ndef format_tool_message(response: str, ai_message: AIMessage):\n return ToolMessage(\n content=response + \"\\nMake all fixes using the writePython tool.\",\n tool_call_id=ai_message.tool_calls[0][\"id\"],\n )\n\n\ndef evaluate(state: State):\n test_cases = state[\"test_cases\"]\n ai_message: AIMessage = state[\"messages\"][-1]\n if not ai_message.tool_calls:\n return {\n \"messages\": [\n HumanMessage(\n content=\"No code submitted. Please try again using the correct python code.\"\n )\n ]\n }\n try:\n code = ai_message.tool_calls[0][\"args\"][\"code\"]\n except Exception as e:\n return {\"messages\": [format_tool_message(repr(e), ai_message)]}\n num_test_cases = len(test_cases)\n succeeded = 0\n test_results = []\n # TODO: Multiprocess\n for test_case in test_cases:\n input_data = test_case[\"inputs\"]\n expected_output = test_case[\"outputs\"]\n test_result = check_correctness(code, input_data, expected_output, timeout)\n test_results.append(test_result)\n if test_result == \"passed\":\n succeeded += 1\n pass_rate = succeeded / num_test_cases if num_test_cases else \"N/A\"\n if pass_rate == 1:\n return {\"status\": \"success\"}\n\n responses = \"\\n\".join(\n [f\"\\n{r}\\n\" for i, r in enumerate(test_results)]\n )\n response = f\"Incorrect submission. Please respond with updated code.\\nPass rate: {succeeded}/{num_test_cases}\\nResults:\\n{responses}\"\n formatted_message = format_tool_message(response, ai_message)\n return {\"messages\": [formatted_message]}"]
+ "source": [
+ "from langchain_core.messages import AIMessage, HumanMessage, ToolMessage\n",
+ "\n",
+ "\n",
+ "# This is the node we will add to the graph.\n",
+ "# Most tool-calling APIs require that the `ToolMessage` contain the ID\n",
+ "# of the\n",
+ "def format_tool_message(response: str, ai_message: AIMessage):\n",
+ " return ToolMessage(\n",
+ " content=response + \"\\nMake all fixes using the writePython tool.\",\n",
+ " tool_call_id=ai_message.tool_calls[0][\"id\"],\n",
+ " )\n",
+ "\n",
+ "\n",
+ "def evaluate(state: State):\n",
+ " test_cases = state[\"test_cases\"]\n",
+ " ai_message: AIMessage = state[\"messages\"][-1]\n",
+ " if not ai_message.tool_calls:\n",
+ " return {\n",
+ " \"messages\": [\n",
+ " HumanMessage(\n",
+ " content=\"No code submitted. Please try again using the correct python code.\"\n",
+ " )\n",
+ " ]\n",
+ " }\n",
+ " try:\n",
+ " code = ai_message.tool_calls[0][\"args\"][\"code\"]\n",
+ " except Exception as e:\n",
+ " return {\"messages\": [format_tool_message(repr(e), ai_message)]}\n",
+ " num_test_cases = len(test_cases)\n",
+ " succeeded = 0\n",
+ " test_results = []\n",
+ " # TODO: Multiprocess\n",
+ " for test_case in test_cases:\n",
+ " input_data = test_case[\"inputs\"]\n",
+ " expected_output = test_case[\"outputs\"]\n",
+ " test_result = check_correctness(code, input_data, expected_output, timeout)\n",
+ " test_results.append(test_result)\n",
+ " if test_result == \"passed\":\n",
+ " succeeded += 1\n",
+ " pass_rate = succeeded / num_test_cases if num_test_cases else \"N/A\"\n",
+ " if pass_rate == 1:\n",
+ " return {\"status\": \"success\"}\n",
+ "\n",
+ " responses = \"\\n\".join(\n",
+ " [f\"\\n{r}\\n\" for i, r in enumerate(test_results)]\n",
+ " )\n",
+ " response = f\"Incorrect submission. Please respond with updated code.\\nPass rate: {succeeded}/{num_test_cases}\\nResults:\\n{responses}\"\n",
+ " formatted_message = format_tool_message(response, ai_message)\n",
+ " return {\"messages\": [formatted_message]}"
+ ]
},
{
"cell_type": "markdown",
@@ -295,7 +543,25 @@
"id": "caf1560e-1517-4229-8a43-186816da6a3a",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.graph import END, StateGraph, START\n\nbuilder = StateGraph(State)\nbuilder.add_node(\"solver\", solver)\nbuilder.add_edge(START, \"solver\")\nbuilder.add_node(\"evaluate\", evaluate)\nbuilder.add_edge(\"solver\", \"evaluate\")\n\n\ndef control_edge(state: State):\n if state.get(\"status\") == \"success\":\n return END\n return \"solver\"\n\n\nbuilder.add_conditional_edges(\"evaluate\", control_edge, {END: END, \"solver\": \"solver\"})\ngraph = builder.compile()"]
+ "source": [
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "builder = StateGraph(State)\n",
+ "builder.add_node(\"solver\", solver)\n",
+ "builder.add_edge(START, \"solver\")\n",
+ "builder.add_node(\"evaluate\", evaluate)\n",
+ "builder.add_edge(\"solver\", \"evaluate\")\n",
+ "\n",
+ "\n",
+ "def control_edge(state: State):\n",
+ " if state.get(\"status\") == \"success\":\n",
+ " return END\n",
+ " return \"solver\"\n",
+ "\n",
+ "\n",
+ "builder.add_conditional_edges(\"evaluate\", control_edge, {END: END, \"solver\": \"solver\"})\n",
+ "graph = builder.compile()"
+ ]
},
{
"cell_type": "code",
@@ -314,7 +580,15 @@
"output_type": "display_data"
}
],
- "source": ["from IPython.display import Image, display\n\ntry:\n display(Image(graph.get_graph().draw_mermaid_png()))\nexcept Exception:\n # This requires some extra dependencies and is optional\n pass"]
+ "source": [
+ "from IPython.display import Image, display\n",
+ "\n",
+ "try:\n",
+ " display(Image(graph.get_graph().draw_mermaid_png()))\n",
+ "except Exception:\n",
+ " # This requires some extra dependencies and is optional\n",
+ " pass"
+ ]
},
{
"cell_type": "markdown",
@@ -393,7 +667,12 @@
]
}
],
- "source": ["input_state = input_states[0].copy()\n# We will reduce the test cases to speed this notebook up\ninput_state[\"test_cases\"] = input_state[\"test_cases\"][:3]\nprint(input_state[\"messages\"][0][1])"]
+ "source": [
+ "input_state = input_states[0].copy()\n",
+ "# We will reduce the test cases to speed this notebook up\n",
+ "input_state[\"test_cases\"] = input_state[\"test_cases\"][:3]\n",
+ "print(input_state[\"messages\"][0][1])"
+ ]
},
{
"cell_type": "markdown",
@@ -455,7 +734,33 @@
]
}
],
- "source": ["from langchain_core.tracers.context import tracing_v2_enabled\nfrom langsmith import Client\n\n\n# We don't need to include all the test cases in our traces.\ndef _hide_test_cases(inputs):\n copied = inputs.copy()\n # These are tens of MB in size. No need to send them up\n copied[\"test_cases\"] = \"...\"\n return copied\n\n\nclient = Client(hide_inputs=_hide_test_cases, hide_outputs=_hide_test_cases)\nwith tracing_v2_enabled(client=client):\n events = graph.stream(input_state)\n for event in events:\n for value in event.values():\n messages = value.get(\"messages\")\n if messages:\n if isinstance(messages, list):\n messages = value[\"messages\"][-1]\n print(\n \"Assistant:\",\n str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n )"]
+ "source": [
+ "from langchain_core.tracers.context import tracing_v2_enabled\n",
+ "from langsmith import Client\n",
+ "\n",
+ "\n",
+ "# We don't need to include all the test cases in our traces.\n",
+ "def _hide_test_cases(inputs):\n",
+ " copied = inputs.copy()\n",
+ " # These are tens of MB in size. No need to send them up\n",
+ " copied[\"test_cases\"] = \"...\"\n",
+ " return copied\n",
+ "\n",
+ "\n",
+ "client = Client(hide_inputs=_hide_test_cases, hide_outputs=_hide_test_cases)\n",
+ "with tracing_v2_enabled(client=client):\n",
+ " events = graph.stream(input_state)\n",
+ " for event in events:\n",
+ " for value in event.values():\n",
+ " messages = value.get(\"messages\")\n",
+ " if messages:\n",
+ " if isinstance(messages, list):\n",
+ " messages = value[\"messages\"][-1]\n",
+ " print(\n",
+ " \"Assistant:\",\n",
+ " str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n",
+ " )"
+ ]
},
{
"cell_type": "markdown",
@@ -501,7 +806,10 @@
"id": "d612dd8d-31af-426c-944b-203acd55ace0",
"metadata": {},
"outputs": [],
- "source": ["%%capture --no-stderr\n%pip install --upgrade --quiet rank_bm25"]
+ "source": [
+ "%%capture --no-stderr\n",
+ "%pip install --upgrade --quiet rank_bm25"
+ ]
},
{
"cell_type": "markdown",
@@ -519,7 +827,29 @@
"id": "16937fef-58b9-4ab2-bbfc-5237aad235ec",
"metadata": {},
"outputs": [],
- "source": ["from typing import Annotated\n\nfrom typing_extensions import TypedDict\n\nfrom langgraph.graph.message import AnyMessage, add_messages\n\n\nclass TestCase(TypedDict):\n inputs: str\n outputs: str\n\n\nclass State(TypedDict):\n # NEW! Candidate for retrieval + formatted fetched examples as \"memory\"\n candidate: AIMessage\n examples: str\n # Repeated from Part 1\n messages: Annotated[list[AnyMessage], add_messages]\n test_cases: list[TestCase]\n runtime_limit: int\n status: str"]
+ "source": [
+ "from typing import Annotated\n",
+ "\n",
+ "from typing_extensions import TypedDict\n",
+ "\n",
+ "from langgraph.graph.message import AnyMessage, add_messages\n",
+ "\n",
+ "\n",
+ "class TestCase(TypedDict):\n",
+ " inputs: str\n",
+ " outputs: str\n",
+ "\n",
+ "\n",
+ "class State(TypedDict):\n",
+ " # NEW! Candidate for retrieval + formatted fetched examples as \"memory\"\n",
+ " candidate: AIMessage\n",
+ " examples: str\n",
+ " # Repeated from Part 1\n",
+ " messages: Annotated[list[AnyMessage], add_messages]\n",
+ " test_cases: list[TestCase]\n",
+ " runtime_limit: int\n",
+ " status: str"
+ ]
},
{
"cell_type": "markdown",
@@ -537,7 +867,40 @@
"id": "25f947a7-15bb-4119-a47e-b5c33ca0a249",
"metadata": {},
"outputs": [],
- "source": ["from langchain import hub\nfrom langchain_anthropic import ChatAnthropic\n\n\nclass Solver:\n def __init__(self, llm: BaseChatModel, prompt: ChatPromptTemplate):\n self.runnable = prompt | llm.bind_tools([writePython])\n\n def __call__(self, state: State) -> dict:\n # Our agent only can see the \"messages\" and will ignore the test info\n inputs = {\"messages\": state[\"messages\"]}\n has_examples = bool(state.get(\"examples\"))\n output_key = \"candidate\" # Used in the draft node\n if has_examples:\n output_key = \"messages\"\n # Used in the solve node\n inputs[\"examples\"] = state[\"examples\"]\n response = self.runnable.invoke(inputs)\n if not response.content:\n return {\n output_key: AIMessage(\n content=\"I'll need to think about this step by step.\"\n )\n }\n return {output_key: response}\n\n\nprompt = hub.pull(\"wfh/usaco-draft-solver\")\nllm = ChatAnthropic(model=\"claude-3-opus-20240229\")\n\ndraft_solver = Solver(llm, prompt.partial(examples=\"\"))\nsolver = Solver(llm, prompt)"]
+ "source": [
+ "from langchain import hub\n",
+ "from langchain_anthropic import ChatAnthropic\n",
+ "\n",
+ "\n",
+ "class Solver:\n",
+ " def __init__(self, llm: BaseChatModel, prompt: ChatPromptTemplate):\n",
+ " self.runnable = prompt | llm.bind_tools([writePython])\n",
+ "\n",
+ " def __call__(self, state: State) -> dict:\n",
+ " # Our agent only can see the \"messages\" and will ignore the test info\n",
+ " inputs = {\"messages\": state[\"messages\"]}\n",
+ " has_examples = bool(state.get(\"examples\"))\n",
+ " output_key = \"candidate\" # Used in the draft node\n",
+ " if has_examples:\n",
+ " output_key = \"messages\"\n",
+ " # Used in the solve node\n",
+ " inputs[\"examples\"] = state[\"examples\"]\n",
+ " response = self.runnable.invoke(inputs)\n",
+ " if not response.content:\n",
+ " return {\n",
+ " output_key: AIMessage(\n",
+ " content=\"I'll need to think about this step by step.\"\n",
+ " )\n",
+ " }\n",
+ " return {output_key: response}\n",
+ "\n",
+ "\n",
+ "prompt = hub.pull(\"wfh/usaco-draft-solver\")\n",
+ "llm = ChatAnthropic(model=\"claude-3-opus-20240229\")\n",
+ "\n",
+ "draft_solver = Solver(llm, prompt.partial(examples=\"\"))\n",
+ "solver = Solver(llm, prompt)"
+ ]
},
{
"cell_type": "markdown",
@@ -555,7 +918,13 @@
"id": "e5e0aa40-79a4-4071-9ad2-9aa2f36599ce",
"metadata": {},
"outputs": [],
- "source": ["# We will test our agent on index 0 (the same as above).\n# Later, we will test on index 2 (the first 'silver difficulty' question)\ntest_indices = [0, 2]\ntrain_ds = [row for i, row in enumerate(ds) if i not in test_indices]\ntest_ds = [row for i, row in enumerate(ds) if i in test_indices]"]
+ "source": [
+ "# We will test our agent on index 0 (the same as above).\n",
+ "# Later, we will test on index 2 (the first 'silver difficulty' question)\n",
+ "test_indices = [0, 2]\n",
+ "train_ds = [row for i, row in enumerate(ds) if i not in test_indices]\n",
+ "test_ds = [row for i, row in enumerate(ds) if i in test_indices]"
+ ]
},
{
"cell_type": "code",
@@ -563,7 +932,25 @@
"id": "96a1ff96-7556-4959-9f54-1ade3bd1c01a",
"metadata": {},
"outputs": [],
- "source": ["from langchain_community.retrievers import BM25Retriever\n\n\ndef format_example(row):\n question = row[\"description\"]\n answer = row[\"solution\"]\n return f\"\"\"\n{question}\n\n\n{answer}\n\"\"\"\n\n\n# Skip our 'test examples' to avoid cheating\n# This is \"simulating\" having seen other in-context examples\nretriever = BM25Retriever.from_texts([format_example(row) for row in train_ds])"]
+ "source": [
+ "from langchain_community.retrievers import BM25Retriever\n",
+ "\n",
+ "\n",
+ "def format_example(row):\n",
+ " question = row[\"description\"]\n",
+ " answer = row[\"solution\"]\n",
+ " return f\"\"\"\n",
+ "{question}\n",
+ "\n",
+ "\n",
+ "{answer}\n",
+ "\"\"\"\n",
+ "\n",
+ "\n",
+ "# Skip our 'test examples' to avoid cheating\n",
+ "# This is \"simulating\" having seen other in-context examples\n",
+ "retriever = BM25Retriever.from_texts([format_example(row) for row in train_ds])"
+ ]
},
{
"cell_type": "markdown",
@@ -580,7 +967,28 @@
"id": "af42962d-c06e-4b6e-96df-72ad48f17617",
"metadata": {},
"outputs": [],
- "source": ["from langchain_core.runnables import RunnableConfig\n\n\ndef retrieve_examples(state: State, config: RunnableConfig):\n top_k = config[\"configurable\"].get(\"k\") or 2\n ai_message: AIMessage = state[\"candidate\"]\n if not ai_message.tool_calls:\n # We err here. To make more robust, you could loop back\n raise ValueError(\"Draft agent did not produce a valid code block\")\n code = ai_message.tool_calls[0][\"args\"][\"code\"]\n examples_str = \"\\n\".join(\n [doc.page_content for doc in retriever.invoke(code)[:top_k]]\n )\n examples_str = f\"\"\"\nYou previously solved the following problems in this competition:\n\n{examples_str}\n\nApproach this new question with similar sophistication.\"\"\"\n return {\"examples\": examples_str}"]
+ "source": [
+ "from langchain_core.runnables import RunnableConfig\n",
+ "\n",
+ "\n",
+ "def retrieve_examples(state: State, config: RunnableConfig):\n",
+ " top_k = config[\"configurable\"].get(\"k\") or 2\n",
+ " ai_message: AIMessage = state[\"candidate\"]\n",
+ " if not ai_message.tool_calls:\n",
+ " # We err here. To make more robust, you could loop back\n",
+ " raise ValueError(\"Draft agent did not produce a valid code block\")\n",
+ " code = ai_message.tool_calls[0][\"args\"][\"code\"]\n",
+ " examples_str = \"\\n\".join(\n",
+ " [doc.page_content for doc in retriever.invoke(code)[:top_k]]\n",
+ " )\n",
+ " examples_str = f\"\"\"\n",
+ "You previously solved the following problems in this competition:\n",
+ "\n",
+ "{examples_str}\n",
+ "\n",
+ "Approach this new question with similar sophistication.\"\"\"\n",
+ " return {\"examples\": examples_str}"
+ ]
},
{
"cell_type": "markdown",
@@ -598,7 +1006,34 @@
"id": "e6e73e85-1232-4848-beba-3139ac7d0a64",
"metadata": {},
"outputs": [],
- "source": ["from langgraph.checkpoint.memory import MemorySaver\nfrom langgraph.graph import END, StateGraph, START\n\nbuilder = StateGraph(State)\nbuilder.add_node(\"draft\", draft_solver)\nbuilder.add_edge(START, \"draft\")\nbuilder.add_node(\"retrieve\", retrieve_examples)\nbuilder.add_node(\"solve\", solver)\nbuilder.add_node(\"evaluate\", evaluate)\n# Add connectivity\nbuilder.add_edge(\"draft\", \"retrieve\")\nbuilder.add_edge(\"retrieve\", \"solve\")\nbuilder.add_edge(\"solve\", \"evaluate\")\n\n\ndef control_edge(state: State):\n if state.get(\"status\") == \"success\":\n return END\n return \"solve\"\n\n\nbuilder.add_conditional_edges(\"evaluate\", control_edge, {END: END, \"solve\": \"solve\"})\n\n\ncheckpointer = MemorySaver()\ngraph = builder.compile(checkpointer=checkpointer)"]
+ "source": [
+ "from langgraph.checkpoint.memory import MemorySaver\n",
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "builder = StateGraph(State)\n",
+ "builder.add_node(\"draft\", draft_solver)\n",
+ "builder.add_edge(START, \"draft\")\n",
+ "builder.add_node(\"retrieve\", retrieve_examples)\n",
+ "builder.add_node(\"solve\", solver)\n",
+ "builder.add_node(\"evaluate\", evaluate)\n",
+ "# Add connectivity\n",
+ "builder.add_edge(\"draft\", \"retrieve\")\n",
+ "builder.add_edge(\"retrieve\", \"solve\")\n",
+ "builder.add_edge(\"solve\", \"evaluate\")\n",
+ "\n",
+ "\n",
+ "def control_edge(state: State):\n",
+ " if state.get(\"status\") == \"success\":\n",
+ " return END\n",
+ " return \"solve\"\n",
+ "\n",
+ "\n",
+ "builder.add_conditional_edges(\"evaluate\", control_edge, {END: END, \"solve\": \"solve\"})\n",
+ "\n",
+ "\n",
+ "checkpointer = MemorySaver()\n",
+ "graph = builder.compile(checkpointer=checkpointer)"
+ ]
},
{
"cell_type": "code",
@@ -617,7 +1052,15 @@
"output_type": "display_data"
}
],
- "source": ["from IPython.display import Image, display\n\ntry:\n display(Image(graph.get_graph().draw_mermaid_png()))\nexcept Exception:\n # This requires some extra dependencies and is optional\n pass"]
+ "source": [
+ "from IPython.display import Image, display\n",
+ "\n",
+ "try:\n",
+ " display(Image(graph.get_graph().draw_mermaid_png()))\n",
+ "except Exception:\n",
+ " # This requires some extra dependencies and is optional\n",
+ " pass"
+ ]
},
{
"cell_type": "markdown",
@@ -650,7 +1093,25 @@
]
}
],
- "source": ["config = {\"configurable\": {\"thread_id\": \"question-recall\", \"k\": 3}}\nwith tracing_v2_enabled(client=client):\n events = graph.stream(input_state, config)\n for event in events:\n for value in event.values():\n messages = value.get(\"messages\")\n if messages:\n if isinstance(messages, list):\n messages = value[\"messages\"][-1]\n print(\n \"Assistant:\",\n str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n )\n elif value.get(\"examples\"):\n print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n elif value.get(\"candidate\"):\n print(str(value[\"candidate\"].content)[:200])"]
+ "source": [
+ "config = {\"configurable\": {\"thread_id\": \"question-recall\", \"k\": 3}}\n",
+ "with tracing_v2_enabled(client=client):\n",
+ " events = graph.stream(input_state, config)\n",
+ " for event in events:\n",
+ " for value in event.values():\n",
+ " messages = value.get(\"messages\")\n",
+ " if messages:\n",
+ " if isinstance(messages, list):\n",
+ " messages = value[\"messages\"][-1]\n",
+ " print(\n",
+ " \"Assistant:\",\n",
+ " str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n",
+ " )\n",
+ " elif value.get(\"examples\"):\n",
+ " print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n",
+ " elif value.get(\"candidate\"):\n",
+ " print(str(value[\"candidate\"].content)[:200])"
+ ]
},
{
"cell_type": "markdown",
@@ -677,7 +1138,10 @@
"output_type": "execute_result"
}
],
- "source": ["checkpoint = graph.get_state(config)\ncheckpoint.values[\"status\"]"]
+ "source": [
+ "checkpoint = graph.get_state(config)\n",
+ "checkpoint.values[\"status\"]"
+ ]
},
{
"cell_type": "markdown",
@@ -706,7 +1170,10 @@
"output_type": "execute_result"
}
],
- "source": ["silver_row = test_ds[1]\nsilver_row[\"problem_level\"]"]
+ "source": [
+ "silver_row = test_ds[1]\n",
+ "silver_row[\"problem_level\"]"
+ ]
},
{
"cell_type": "code",
@@ -764,7 +1231,33 @@
]
}
],
- "source": ["silver_input = {\n \"messages\": [(\"user\", silver_row[\"description\"])],\n \"test_cases\": silver_row[\"test_cases\"],\n \"runtime_limit\": silver_row[\"runtime_limit\"],\n \"status\": \"in_progress\",\n}\n\n\nconfig = {\"configurable\": {\"thread_id\": \"silver-question-1\", \"k\": 2}}\nwith tracing_v2_enabled(client=client):\n events = graph.stream(silver_input, config)\n for event in events:\n for value in event.values():\n messages = value.get(\"messages\")\n if messages:\n if isinstance(messages, list):\n messages = value[\"messages\"][-1]\n print(\n \"Assistant:\",\n str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n )\n elif value.get(\"examples\"):\n print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n elif value.get(\"candidate\"):\n print(str(value[\"candidate\"].content)[:200])"]
+ "source": [
+ "silver_input = {\n",
+ " \"messages\": [(\"user\", silver_row[\"description\"])],\n",
+ " \"test_cases\": silver_row[\"test_cases\"],\n",
+ " \"runtime_limit\": silver_row[\"runtime_limit\"],\n",
+ " \"status\": \"in_progress\",\n",
+ "}\n",
+ "\n",
+ "\n",
+ "config = {\"configurable\": {\"thread_id\": \"silver-question-1\", \"k\": 2}}\n",
+ "with tracing_v2_enabled(client=client):\n",
+ " events = graph.stream(silver_input, config)\n",
+ " for event in events:\n",
+ " for value in event.values():\n",
+ " messages = value.get(\"messages\")\n",
+ " if messages:\n",
+ " if isinstance(messages, list):\n",
+ " messages = value[\"messages\"][-1]\n",
+ " print(\n",
+ " \"Assistant:\",\n",
+ " str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n",
+ " )\n",
+ " elif value.get(\"examples\"):\n",
+ " print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n",
+ " elif value.get(\"candidate\"):\n",
+ " print(str(value[\"candidate\"].content)[:200])"
+ ]
},
{
"cell_type": "markdown",
@@ -809,7 +1302,36 @@
"id": "3c6456ba-363c-4133-8631-6dabb042b6ce",
"metadata": {},
"outputs": [],
- "source": ["# This is all the same as before\nfrom langgraph.checkpoint.memory import MemorySaver\nfrom langgraph.graph import END, StateGraph, START\n\nbuilder = StateGraph(State)\nprompt = hub.pull(\"wfh/usaco-draft-solver\")\nllm = ChatAnthropic(model=\"claude-3-opus-20240229\", max_tokens_to_sample=4000)\n\ndraft_solver = Solver(llm, prompt.partial(examples=\"\"))\nbuilder.add_node(\"draft\", draft_solver)\nbuilder.add_edge(START, \"draft\")\nbuilder.add_node(\"retrieve\", retrieve_examples)\nsolver = Solver(llm, prompt)\nbuilder.add_node(\"solve\", solver)\nbuilder.add_node(\"evaluate\", evaluate)\nbuilder.add_edge(\"draft\", \"retrieve\")\nbuilder.add_edge(\"retrieve\", \"solve\")\nbuilder.add_edge(\"solve\", \"evaluate\")\n\n\ndef control_edge(state: State):\n if state.get(\"status\") == \"success\":\n return END\n return \"solve\"\n\n\nbuilder.add_conditional_edges(\"evaluate\", control_edge, {END: END, \"solve\": \"solve\"})\ncheckpointer = MemorySaver()"]
+ "source": [
+ "# This is all the same as before\n",
+ "from langgraph.checkpoint.memory import MemorySaver\n",
+ "from langgraph.graph import END, StateGraph, START\n",
+ "\n",
+ "builder = StateGraph(State)\n",
+ "prompt = hub.pull(\"wfh/usaco-draft-solver\")\n",
+ "llm = ChatAnthropic(model=\"claude-3-opus-20240229\", max_tokens_to_sample=4000)\n",
+ "\n",
+ "draft_solver = Solver(llm, prompt.partial(examples=\"\"))\n",
+ "builder.add_node(\"draft\", draft_solver)\n",
+ "builder.add_edge(START, \"draft\")\n",
+ "builder.add_node(\"retrieve\", retrieve_examples)\n",
+ "solver = Solver(llm, prompt)\n",
+ "builder.add_node(\"solve\", solver)\n",
+ "builder.add_node(\"evaluate\", evaluate)\n",
+ "builder.add_edge(\"draft\", \"retrieve\")\n",
+ "builder.add_edge(\"retrieve\", \"solve\")\n",
+ "builder.add_edge(\"solve\", \"evaluate\")\n",
+ "\n",
+ "\n",
+ "def control_edge(state: State):\n",
+ " if state.get(\"status\") == \"success\":\n",
+ " return END\n",
+ " return \"solve\"\n",
+ "\n",
+ "\n",
+ "builder.add_conditional_edges(\"evaluate\", control_edge, {END: END, \"solve\": \"solve\"})\n",
+ "checkpointer = MemorySaver()"
+ ]
},
{
"cell_type": "markdown",
@@ -825,7 +1347,13 @@
"id": "461c13ba-01cc-44e1-b837-6a64d03069d9",
"metadata": {},
"outputs": [],
- "source": ["graph = builder.compile(\n checkpointer=checkpointer,\n # New: this tells the graph to break any time it goes to the \"human\" node\n interrupt_after=[\"evaluate\"],\n)"]
+ "source": [
+ "graph = builder.compile(\n",
+ " checkpointer=checkpointer,\n",
+ " # New: this tells the graph to break any time it goes to the \"human\" node\n",
+ " interrupt_after=[\"evaluate\"],\n",
+ ")"
+ ]
},
{
"cell_type": "code",
@@ -844,7 +1372,15 @@
"output_type": "display_data"
}
],
- "source": ["from IPython.display import Image, display\n\ntry:\n display(Image(graph.get_graph().draw_mermaid_png()))\nexcept Exception:\n # This requires some extra dependencies and is optional\n pass"]
+ "source": [
+ "from IPython.display import Image, display\n",
+ "\n",
+ "try:\n",
+ " display(Image(graph.get_graph().draw_mermaid_png()))\n",
+ "except Exception:\n",
+ " # This requires some extra dependencies and is optional\n",
+ " pass"
+ ]
},
{
"cell_type": "markdown",
@@ -879,7 +1415,25 @@
]
}
],
- "source": ["config = {\"configurable\": {\"thread_id\": \"silver-hl-1\", \"k\": 2}}\nwith tracing_v2_enabled(client=client):\n events = graph.stream(silver_input, config)\n for event in events:\n for value in event.values():\n messages = value.get(\"messages\")\n if messages:\n if isinstance(messages, list):\n messages = value[\"messages\"][-1]\n print(\n \"Assistant:\",\n str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n )\n elif value.get(\"examples\"):\n print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n elif value.get(\"candidate\"):\n print(str(value[\"candidate\"].content)[:200])"]
+ "source": [
+ "config = {\"configurable\": {\"thread_id\": \"silver-hl-1\", \"k\": 2}}\n",
+ "with tracing_v2_enabled(client=client):\n",
+ " events = graph.stream(silver_input, config)\n",
+ " for event in events:\n",
+ " for value in event.values():\n",
+ " messages = value.get(\"messages\")\n",
+ " if messages:\n",
+ " if isinstance(messages, list):\n",
+ " messages = value[\"messages\"][-1]\n",
+ " print(\n",
+ " \"Assistant:\",\n",
+ " str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n",
+ " )\n",
+ " elif value.get(\"examples\"):\n",
+ " print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n",
+ " elif value.get(\"candidate\"):\n",
+ " print(str(value[\"candidate\"].content)[:200])"
+ ]
},
{
"cell_type": "markdown",
@@ -962,7 +1516,10 @@
]
}
],
- "source": ["snapshot = graph.get_state(config)\nprint(snapshot.values[\"messages\"][0].content)"]
+ "source": [
+ "snapshot = graph.get_state(config)\n",
+ "print(snapshot.values[\"messages\"][0].content)"
+ ]
},
{
"cell_type": "markdown",
@@ -1042,7 +1599,12 @@
]
}
],
- "source": ["snapshot = graph.get_state(config)\nprint(snapshot.values[\"messages\"][-2].content[0][\"text\"])\nprint(\"\\n\\nCode:\\n\\n\")\nprint(snapshot.values[\"messages\"][-2].tool_calls[0][\"args\"][\"code\"])"]
+ "source": [
+ "snapshot = graph.get_state(config)\n",
+ "print(snapshot.values[\"messages\"][-2].content[0][\"text\"])\n",
+ "print(\"\\n\\nCode:\\n\\n\")\n",
+ "print(snapshot.values[\"messages\"][-2].tool_calls[0][\"args\"][\"code\"])"
+ ]
},
{
"cell_type": "code",
@@ -1071,7 +1633,9 @@
]
}
],
- "source": ["print(snapshot.values[\"messages\"][-1].content[:200])"]
+ "source": [
+ "print(snapshot.values[\"messages\"][-1].content[:200])"
+ ]
},
{
"cell_type": "markdown",
@@ -1091,7 +1655,51 @@
"id": "b10fcbc9-6dd4-41ad-98f7-1cf1685035e6",
"metadata": {},
"outputs": [],
- "source": ["updated_config = graph.update_state(\n config,\n values={\n \"messages\": [\n (\n \"user\",\n \"\"\"Consider breaking down the algorithm into separate parts: reading inputs, detecting cycles using the tortoise and hare algorithm, and determining Bessie's final position by skipping ahead K steps.\n\nRead the inputs into three arrays:\n- Two arrays L and R for the ports (adjust for 0-based indexing)\n- A third array S for the direction sequence\n\nOptimize by multiplying K by M before the main loop to convert the number of repetitions into the total number of steps.\n\nUse the tortoise and hare algorithm to detect the cycle:\n- Define a helper function get_next(v) that returns the next position and direction index\n- Initialize two pointers s0 and s1 to (0, 0)\n- In each iteration:\n - Move s0 by 1 step and s1 by 2 steps using get_next()\n - If s0 equals s1, decrement K by 1 and break out of the loop\n - Otherwise, decrement K by 1\n- After the loop, if K is not 0, there is a cycle\n\nTo find the cycle length:\n- Initialize a counter variable rho to 1\n- Move s0 by 1 step using get_next()\n- Enter a loop:\n - Move s0 by 1 step using get_next()\n - Increment rho\n - If s0 equals s1, break out of the loop\n\nSkip ahead by reducing K modulo rho.\n\nSimulate the remaining steps:\n- While K > 0, move s0 to the next position using get_next() and decrement K\n\nPrint the final position (converted to 1-based indexing).\n\nPay close attention to the initialization and movement of pointers during cycle detection and length calculation. Ensure that the logic is correct and handles all cases accurately.\"\"\",\n )\n ]\n },\n)"]
+ "source": [
+ "updated_config = graph.update_state(\n",
+ " config,\n",
+ " values={\n",
+ " \"messages\": [\n",
+ " (\n",
+ " \"user\",\n",
+ " \"\"\"Consider breaking down the algorithm into separate parts: reading inputs, detecting cycles using the tortoise and hare algorithm, and determining Bessie's final position by skipping ahead K steps.\n",
+ "\n",
+ "Read the inputs into three arrays:\n",
+ "- Two arrays L and R for the ports (adjust for 0-based indexing)\n",
+ "- A third array S for the direction sequence\n",
+ "\n",
+ "Optimize by multiplying K by M before the main loop to convert the number of repetitions into the total number of steps.\n",
+ "\n",
+ "Use the tortoise and hare algorithm to detect the cycle:\n",
+ "- Define a helper function get_next(v) that returns the next position and direction index\n",
+ "- Initialize two pointers s0 and s1 to (0, 0)\n",
+ "- In each iteration:\n",
+ " - Move s0 by 1 step and s1 by 2 steps using get_next()\n",
+ " - If s0 equals s1, decrement K by 1 and break out of the loop\n",
+ " - Otherwise, decrement K by 1\n",
+ "- After the loop, if K is not 0, there is a cycle\n",
+ "\n",
+ "To find the cycle length:\n",
+ "- Initialize a counter variable rho to 1\n",
+ "- Move s0 by 1 step using get_next()\n",
+ "- Enter a loop:\n",
+ " - Move s0 by 1 step using get_next()\n",
+ " - Increment rho\n",
+ " - If s0 equals s1, break out of the loop\n",
+ "\n",
+ "Skip ahead by reducing K modulo rho.\n",
+ "\n",
+ "Simulate the remaining steps:\n",
+ "- While K > 0, move s0 to the next position using get_next() and decrement K\n",
+ "\n",
+ "Print the final position (converted to 1-based indexing).\n",
+ "\n",
+ "Pay close attention to the initialization and movement of pointers during cycle detection and length calculation. Ensure that the logic is correct and handles all cases accurately.\"\"\",\n",
+ " )\n",
+ " ]\n",
+ " },\n",
+ ")"
+ ]
},
{
"cell_type": "markdown",
@@ -1118,7 +1726,9 @@
"output_type": "execute_result"
}
],
- "source": ["graph.get_state(config).values[\"messages\"][-1]"]
+ "source": [
+ "graph.get_state(config).values[\"messages\"][-1]"
+ ]
},
{
"cell_type": "markdown",
@@ -1145,7 +1755,29 @@
]
}
],
- "source": ["num_trials = 1\nwith tracing_v2_enabled(client=client):\n for _ in range(num_trials):\n events = graph.stream(None, updated_config)\n for event in events:\n for value in event.values():\n messages = value.get(\"messages\")\n if messages:\n if isinstance(messages, list):\n messages = value[\"messages\"][-1]\n print(\n \"Assistant:\",\n str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n )\n elif value.get(\"examples\"):\n print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n elif value.get(\"candidate\"):\n print(str(value[\"candidate\"].content)[:200])\n if graph.get_state(config).values[\"status\"] == \"success\":\n break\n print(\"Continuing...\")"]
+ "source": [
+ "num_trials = 1\n",
+ "with tracing_v2_enabled(client=client):\n",
+ " for _ in range(num_trials):\n",
+ " events = graph.stream(None, updated_config)\n",
+ " for event in events:\n",
+ " for value in event.values():\n",
+ " messages = value.get(\"messages\")\n",
+ " if messages:\n",
+ " if isinstance(messages, list):\n",
+ " messages = value[\"messages\"][-1]\n",
+ " print(\n",
+ " \"Assistant:\",\n",
+ " str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n",
+ " )\n",
+ " elif value.get(\"examples\"):\n",
+ " print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n",
+ " elif value.get(\"candidate\"):\n",
+ " print(str(value[\"candidate\"].content)[:200])\n",
+ " if graph.get_state(config).values[\"status\"] == \"success\":\n",
+ " break\n",
+ " print(\"Continuing...\")"
+ ]
},
{
"cell_type": "code",
@@ -1153,7 +1785,9 @@
"id": "20ee7535-1bc8-4105-87c4-0e7a89a011ff",
"metadata": {},
"outputs": [],
- "source": ["most_recent_state = list(graph.get_state_history(config))[0]"]
+ "source": [
+ "most_recent_state = list(graph.get_state_history(config))[0]"
+ ]
},
{
"cell_type": "markdown",
@@ -1231,7 +1865,14 @@
]
}
],
- "source": ["snapshot = graph.get_state(most_recent_state.config)\nai_message = snapshot.values[\"messages\"][-2]\nif ai_message.content:\n print(ai_message.content)\nprint(\"\\n\\nCode:\\n\\n\")\nprint(ai_message.tool_calls[0][\"args\"][\"code\"] if ai_message.tool_calls else \"N/A\")"]
+ "source": [
+ "snapshot = graph.get_state(most_recent_state.config)\n",
+ "ai_message = snapshot.values[\"messages\"][-2]\n",
+ "if ai_message.content:\n",
+ " print(ai_message.content)\n",
+ "print(\"\\n\\nCode:\\n\\n\")\n",
+ "print(ai_message.tool_calls[0][\"args\"][\"code\"] if ai_message.tool_calls else \"N/A\")"
+ ]
},
{
"cell_type": "code",
@@ -1262,7 +1903,9 @@
]
}
],
- "source": ["print(snapshot.values[\"messages\"][-1].content[:200])"]
+ "source": [
+ "print(snapshot.values[\"messages\"][-1].content[:200])"
+ ]
},
{
"cell_type": "markdown",
@@ -1280,7 +1923,24 @@
"id": "6eb46517-cdd1-4716-8a9e-df72cdd9ba67",
"metadata": {},
"outputs": [],
- "source": ["updated_config = graph.update_state(\n updated_config,\n values={\n \"messages\": [\n (\n \"user\",\n \"\"\"That's better, but you're still getting some errors. Let's double check some things:\n \n1. When calculating the cycle length, make sure the initialization and movement of the pointers is correct. Double-check the logic there and see if you can spot any discrepancies.\n2. Check the condition for whether there's a cycle after the main loop to ensure it covers all cases, like if K becomes 0 in the last iteration.\n\nThink step by step through youur implementation and update using the writePython tool.\"\"\",\n )\n ]\n },\n)"]
+ "source": [
+ "updated_config = graph.update_state(\n",
+ " updated_config,\n",
+ " values={\n",
+ " \"messages\": [\n",
+ " (\n",
+ " \"user\",\n",
+ " \"\"\"That's better, but you're still getting some errors. Let's double check some things:\n",
+ " \n",
+ "1. When calculating the cycle length, make sure the initialization and movement of the pointers is correct. Double-check the logic there and see if you can spot any discrepancies.\n",
+ "2. Check the condition for whether there's a cycle after the main loop to ensure it covers all cases, like if K becomes 0 in the last iteration.\n",
+ "\n",
+ "Think step by step through youur implementation and update using the writePython tool.\"\"\",\n",
+ " )\n",
+ " ]\n",
+ " },\n",
+ ")"
+ ]
},
{
"cell_type": "markdown",
@@ -1304,7 +1964,29 @@
]
}
],
- "source": ["num_trials = 2\nwith tracing_v2_enabled(client=client):\n for _ in range(num_trials):\n events = graph.stream(None, updated_config)\n for event in events:\n for value in event.values():\n messages = value.get(\"messages\")\n if messages:\n if isinstance(messages, list):\n messages = value[\"messages\"][-1]\n print(\n \"Assistant:\",\n str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n )\n elif value.get(\"examples\"):\n print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n elif value.get(\"candidate\"):\n print(str(value[\"candidate\"].content)[:200])\n if graph.get_state(config).values[\"status\"] == \"success\":\n break\n print(\"Continuing...\")"]
+ "source": [
+ "num_trials = 2\n",
+ "with tracing_v2_enabled(client=client):\n",
+ " for _ in range(num_trials):\n",
+ " events = graph.stream(None, updated_config)\n",
+ " for event in events:\n",
+ " for value in event.values():\n",
+ " messages = value.get(\"messages\")\n",
+ " if messages:\n",
+ " if isinstance(messages, list):\n",
+ " messages = value[\"messages\"][-1]\n",
+ " print(\n",
+ " \"Assistant:\",\n",
+ " str(messages.content).replace(\"\\n\", \"\\\\n\")[:50],\n",
+ " )\n",
+ " elif value.get(\"examples\"):\n",
+ " print(\"Retrieved examples:\\n\\n\", value[\"examples\"][:100] + \"...\")\n",
+ " elif value.get(\"candidate\"):\n",
+ " print(str(value[\"candidate\"].content)[:200])\n",
+ " if graph.get_state(config).values[\"status\"] == \"success\":\n",
+ " break\n",
+ " print(\"Continuing...\")"
+ ]
},
{
"cell_type": "markdown",
@@ -1328,7 +2010,10 @@
]
}
],
- "source": ["snapshot = graph.get_state(config)\nprint(snapshot.values[\"status\"])"]
+ "source": [
+ "snapshot = graph.get_state(config)\n",
+ "print(snapshot.values[\"status\"])"
+ ]
},
{
"cell_type": "markdown",
@@ -1354,14 +2039,6 @@
"\n",
"LLMs are not capable of solving all these problems autonomously, but through better prompting and clever engineering, you can create a system that is able to more reliably arrive at the proper solution."
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c71b4ba7-96ba-4643-b2de-eb2acdf4daba",
- "metadata": {},
- "outputs": [],
- "source": [""]
}
],
"metadata": {