WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-08-17 21:25:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

[Docs] Update notebooks to use START (#902)

Browse Source
This commit is contained in:
William FH
2024-07-01 21:36:34 -07:00
committed by GitHub
parent 267f5e5234
commit 727e63c01e
67 changed files with 1059 additions and 20258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/rag/langgraph_adaptive_rag.ipynb
+17 -550
View File
File diff suppressed because one or more lines are too long
examples/rag/langgraph_adaptive_rag_cohere.ipynb
+18 -596
View File
File diff suppressed because one or more lines are too long
examples/rag/langgraph_adaptive_rag_local.ipynb
+16 -482
View File
File diff suppressed because one or more lines are too long
examples/rag/langgraph_agentic_rag.ipynb
+10 -307
View File
@@ -20,10 +20,7 @@
"id": "969fb438",
"metadata": {},
"outputs": [],
"source": [
"%%capture --no-stderr\n",
"%pip install -U --quiet langchain-community tiktoken langchain-openai langchainhub chromadb langchain langgraph langchain-text-splitters"
]
"source": ["%%capture --no-stderr\n%pip install -U --quiet langchain-community tiktoken langchain-openai langchainhub chromadb langchain langgraph langchain-text-splitters"]
},
{
"cell_type": "code",
@@ -31,22 +28,7 @@
"id": "e4958a8c",
"metadata": {},
"outputs": [],
"source": [
"import getpass\n",
"import os\n",
"\n",
"\n",
"def _set_env(key: str):\n",
" if key not in os.environ:\n",
" os.environ[key] = getpass.getpass(f\"{key}:\")\n",
"\n",
"\n",
"_set_env(\"OPENAI_API_KEY\")\n",
"\n",
"# (Optional) For tracing\n",
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"_set_env(\"LANGCHAIN_API_KEY\")"
]
"source": ["import getpass\nimport os\n\n\ndef _set_env(key: str):\n if key not in os.environ:\n os.environ[key] = getpass.getpass(f\"{key}:\")\n\n\n_set_env(\"OPENAI_API_KEY\")\n\n# (Optional) For tracing\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n_set_env(\"LANGCHAIN_API_KEY\")"]
},
{
"cell_type": "markdown",
@@ -64,34 +46,7 @@
"id": "e50c9efe-4abe-42fa-b35a-05eeeede9ec6",
"metadata": {},
"outputs": [],
"source": [
"from langchain_community.document_loaders import WebBaseLoader\n",
"from langchain_community.vectorstores import Chroma\n",
"from langchain_openai import OpenAIEmbeddings\n",
"from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
"\n",
"urls = [\n",
" \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
" \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
" \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
"]\n",
"\n",
"docs = [WebBaseLoader(url).load() for url in urls]\n",
"docs_list = [item for sublist in docs for item in sublist]\n",
"\n",
"text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
" chunk_size=100, chunk_overlap=50\n",
")\n",
"doc_splits = text_splitter.split_documents(docs_list)\n",
"\n",
"# Add to vectorDB\n",
"vectorstore = Chroma.from_documents(\n",
" documents=doc_splits,\n",
" collection_name=\"rag-chroma\",\n",
" embedding=OpenAIEmbeddings(),\n",
")\n",
"retriever = vectorstore.as_retriever()"
]
"source": ["from langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_openai import OpenAIEmbeddings\nfrom langchain_text_splitters import RecursiveCharacterTextSplitter\n\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=100, chunk_overlap=50\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorDB\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=OpenAIEmbeddings(),\n)\nretriever = vectorstore.as_retriever()"]
},
{
"cell_type": "markdown",
@@ -107,17 +62,7 @@
"id": "0b97bdd8-d7e3-444d-ac96-5ef4725f9048",
"metadata": {},
"outputs": [],
"source": [
"from langchain.tools.retriever import create_retriever_tool\n",
"\n",
"retriever_tool = create_retriever_tool(\n",
" retriever,\n",
" \"retrieve_blog_posts\",\n",
" \"Search and return information about Lilian Weng blog posts on LLM agents, prompt engineering, and adversarial attacks on LLMs.\",\n",
")\n",
"\n",
"tools = [retriever_tool]"
]
"source": ["from langchain.tools.retriever import create_retriever_tool\n\nretriever_tool = create_retriever_tool(\n retriever,\n \"retrieve_blog_posts\",\n \"Search and return information about Lilian Weng blog posts on LLM agents, prompt engineering, and adversarial attacks on LLMs.\",\n)\n\ntools = [retriever_tool]"]
},
{
"cell_type": "markdown",
@@ -141,19 +86,7 @@
"id": "0e378706-47d5-425a-8ba0-57b9acffbd0c",
"metadata": {},
"outputs": [],
"source": [
"from typing import Annotated, Sequence, TypedDict\n",
"\n",
"from langchain_core.messages import BaseMessage\n",
"\n",
"from langgraph.graph.message import add_messages\n",
"\n",
"\n",
"class AgentState(TypedDict):\n",
" # The add_messages function defines how an update should be processed\n",
" # Default is to replace. add_messages says \"append\"\n",
" messages: Annotated[Sequence[BaseMessage], add_messages]"
]
"source": ["from typing import Annotated, Sequence, TypedDict\n\nfrom langchain_core.messages import BaseMessage\n\nfrom langgraph.graph.message import add_messages\n\n\nclass AgentState(TypedDict):\n # The add_messages function defines how an update should be processed\n # Default is to replace. add_messages says \"append\"\n messages: Annotated[Sequence[BaseMessage], add_messages]"]
},
{
"attachments": {
@@ -196,174 +129,7 @@
]
}
],
"source": [
"from typing import Annotated, Literal, Sequence, TypedDict\n",
"\n",
"from langchain import hub\n",
"from langchain_core.messages import BaseMessage, HumanMessage\n",
"from langchain_core.output_parsers import StrOutputParser\n",
"from langchain_core.prompts import PromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"from langgraph.prebuilt import tools_condition\n",
"\n",
"### Edges\n",
"\n",
"\n",
"def grade_documents(state) -> Literal[\"generate\", \"rewrite\"]:\n",
" \"\"\"\n",
" Determines whether the retrieved documents are relevant to the question.\n",
"\n",
" Args:\n",
" state (messages): The current state\n",
"\n",
" Returns:\n",
" str: A decision for whether the documents are relevant or not\n",
" \"\"\"\n",
"\n",
" print(\"---CHECK RELEVANCE---\")\n",
"\n",
" # Data model\n",
" class grade(BaseModel):\n",
" \"\"\"Binary score for relevance check.\"\"\"\n",
"\n",
" binary_score: str = Field(description=\"Relevance score 'yes' or 'no'\")\n",
"\n",
" # LLM\n",
" model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
"\n",
" # LLM with tool and validation\n",
" llm_with_tool = model.with_structured_output(grade)\n",
"\n",
" # Prompt\n",
" prompt = PromptTemplate(\n",
" template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
" Here is the retrieved document: \\n\\n {context} \\n\\n\n",
" Here is the user question: {question} \\n\n",
" If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
" Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\",\n",
" input_variables=[\"context\", \"question\"],\n",
" )\n",
"\n",
" # Chain\n",
" chain = prompt | llm_with_tool\n",
"\n",
" messages = state[\"messages\"]\n",
" last_message = messages[-1]\n",
"\n",
" question = messages[0].content\n",
" docs = last_message.content\n",
"\n",
" scored_result = chain.invoke({\"question\": question, \"context\": docs})\n",
"\n",
" score = scored_result.binary_score\n",
"\n",
" if score == \"yes\":\n",
" print(\"---DECISION: DOCS RELEVANT---\")\n",
" return \"generate\"\n",
"\n",
" else:\n",
" print(\"---DECISION: DOCS NOT RELEVANT---\")\n",
" print(score)\n",
" return \"rewrite\"\n",
"\n",
"\n",
"### Nodes\n",
"\n",
"\n",
"def agent(state):\n",
" \"\"\"\n",
" Invokes the agent model to generate a response based on the current state. Given\n",
" the question, it will decide to retrieve using the retriever tool, or simply end.\n",
"\n",
" Args:\n",
" state (messages): The current state\n",
"\n",
" Returns:\n",
" dict: The updated state with the agent response appended to messages\n",
" \"\"\"\n",
" print(\"---CALL AGENT---\")\n",
" messages = state[\"messages\"]\n",
" model = ChatOpenAI(temperature=0, streaming=True, model=\"gpt-4-turbo\")\n",
" model = model.bind_tools(tools)\n",
" response = model.invoke(messages)\n",
" # We return a list, because this will get added to the existing list\n",
" return {\"messages\": [response]}\n",
"\n",
"\n",
"def rewrite(state):\n",
" \"\"\"\n",
" Transform the query to produce a better question.\n",
"\n",
" Args:\n",
" state (messages): The current state\n",
"\n",
" Returns:\n",
" dict: The updated state with re-phrased question\n",
" \"\"\"\n",
"\n",
" print(\"---TRANSFORM QUERY---\")\n",
" messages = state[\"messages\"]\n",
" question = messages[0].content\n",
"\n",
" msg = [\n",
" HumanMessage(\n",
" content=f\"\"\" \\n \n",
" Look at the input and try to reason about the underlying semantic intent / meaning. \\n \n",
" Here is the initial question:\n",
" \\n ------- \\n\n",
" {question} \n",
" \\n ------- \\n\n",
" Formulate an improved question: \"\"\",\n",
" )\n",
" ]\n",
"\n",
" # Grader\n",
" model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
" response = model.invoke(msg)\n",
" return {\"messages\": [response]}\n",
"\n",
"\n",
"def generate(state):\n",
" \"\"\"\n",
" Generate answer\n",
"\n",
" Args:\n",
" state (messages): The current state\n",
"\n",
" Returns:\n",
" dict: The updated state with re-phrased question\n",
" \"\"\"\n",
" print(\"---GENERATE---\")\n",
" messages = state[\"messages\"]\n",
" question = messages[0].content\n",
" last_message = messages[-1]\n",
"\n",
" question = messages[0].content\n",
" docs = last_message.content\n",
"\n",
" # Prompt\n",
" prompt = hub.pull(\"rlm/rag-prompt\")\n",
"\n",
" # LLM\n",
" llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0, streaming=True)\n",
"\n",
" # Post-processing\n",
" def format_docs(docs):\n",
" return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
"\n",
" # Chain\n",
" rag_chain = prompt | llm | StrOutputParser()\n",
"\n",
" # Run\n",
" response = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
" return {\"messages\": [response]}\n",
"\n",
"\n",
"print(\"*\" * 20 + \"Prompt[rlm/rag-prompt]\" + \"*\" * 20)\n",
"prompt = hub.pull(\"rlm/rag-prompt\").pretty_print() # Show what the prompt looks like"
]
"source": ["from typing import Annotated, Literal, Sequence, TypedDict\n\nfrom langchain import hub\nfrom langchain_core.messages import BaseMessage, HumanMessage\nfrom langchain_core.output_parsers import StrOutputParser\nfrom langchain_core.prompts import PromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\nfrom langgraph.prebuilt import tools_condition\n\n### Edges\n\n\ndef grade_documents(state) -> Literal[\"generate\", \"rewrite\"]:\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (messages): The current state\n\n Returns:\n str: A decision for whether the documents are relevant or not\n \"\"\"\n\n print(\"---CHECK RELEVANCE---\")\n\n # Data model\n class grade(BaseModel):\n \"\"\"Binary score for relevance check.\"\"\"\n\n binary_score: str = Field(description=\"Relevance score 'yes' or 'no'\")\n\n # LLM\n model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n\n # LLM with tool and validation\n llm_with_tool = model.with_structured_output(grade)\n\n # Prompt\n prompt = PromptTemplate(\n template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n Here is the retrieved document: \\n\\n {context} \\n\\n\n Here is the user question: {question} \\n\n If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\",\n input_variables=[\"context\", \"question\"],\n )\n\n # Chain\n chain = prompt | llm_with_tool\n\n messages = state[\"messages\"]\n last_message = messages[-1]\n\n question = messages[0].content\n docs = last_message.content\n\n scored_result = chain.invoke({\"question\": question, \"context\": docs})\n\n score = scored_result.binary_score\n\n if score == \"yes\":\n print(\"---DECISION: DOCS RELEVANT---\")\n return \"generate\"\n\n else:\n print(\"---DECISION: DOCS NOT RELEVANT---\")\n print(score)\n return \"rewrite\"\n\n\n### Nodes\n\n\ndef agent(state):\n \"\"\"\n Invokes the agent model to generate a response based on the current state. Given\n the question, it will decide to retrieve using the retriever tool, or simply end.\n\n Args:\n state (messages): The current state\n\n Returns:\n dict: The updated state with the agent response appended to messages\n \"\"\"\n print(\"---CALL AGENT---\")\n messages = state[\"messages\"]\n model = ChatOpenAI(temperature=0, streaming=True, model=\"gpt-4-turbo\")\n model = model.bind_tools(tools)\n response = model.invoke(messages)\n # We return a list, because this will get added to the existing list\n return {\"messages\": [response]}\n\n\ndef rewrite(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (messages): The current state\n\n Returns:\n dict: The updated state with re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n messages = state[\"messages\"]\n question = messages[0].content\n\n msg = [\n HumanMessage(\n content=f\"\"\" \\n \n Look at the input and try to reason about the underlying semantic intent / meaning. \\n \n Here is the initial question:\n \\n ------- \\n\n {question} \n \\n ------- \\n\n Formulate an improved question: \"\"\",\n )\n ]\n\n # Grader\n model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n response = model.invoke(msg)\n return {\"messages\": [response]}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (messages): The current state\n\n Returns:\n dict: The updated state with re-phrased question\n \"\"\"\n print(\"---GENERATE---\")\n messages = state[\"messages\"]\n question = messages[0].content\n last_message = messages[-1]\n\n question = messages[0].content\n docs = last_message.content\n\n # Prompt\n prompt = hub.pull(\"rlm/rag-prompt\")\n\n # LLM\n llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0, streaming=True)\n\n # Post-processing\n def format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n # Chain\n rag_chain = prompt | llm | StrOutputParser()\n\n # Run\n response = rag_chain.invoke({\"context\": docs, \"question\": question})\n return {\"messages\": [response]}\n\n\nprint(\"*\" * 20 + \"Prompt[rlm/rag-prompt]\" + \"*\" * 20)\nprompt = hub.pull(\"rlm/rag-prompt\").pretty_print() # Show what the prompt looks like"]
},
{
"cell_type": "markdown",
@@ -384,48 +150,7 @@
"id": "8718a37f-83c2-4f16-9850-e61e0f49c3d4",
"metadata": {},
"outputs": [],
"source": [
"from langgraph.graph import END, StateGraph\n",
"from langgraph.prebuilt import ToolNode\n",
"\n",
"# Define a new graph\n",
"workflow = StateGraph(AgentState)\n",
"\n",
"# Define the nodes we will cycle between\n",
"workflow.add_node(\"agent\", agent) # agent\n",
"retrieve = ToolNode([retriever_tool])\n",
"workflow.add_node(\"retrieve\", retrieve) # retrieval\n",
"workflow.add_node(\"rewrite\", rewrite) # Re-writing the question\n",
"workflow.add_node(\n",
" \"generate\", generate\n",
") # Generating a response after we know the documents are relevant\n",
"# Call agent node to decide to retrieve or not\n",
"workflow.set_entry_point(\"agent\")\n",
"\n",
"# Decide whether to retrieve\n",
"workflow.add_conditional_edges(\n",
" \"agent\",\n",
" # Assess agent decision\n",
" tools_condition,\n",
" {\n",
" # Translate the condition outputs to nodes in our graph\n",
" \"tools\": \"retrieve\",\n",
" END: END,\n",
" },\n",
")\n",
"\n",
"# Edges taken after the `action` node is called.\n",
"workflow.add_conditional_edges(\n",
" \"retrieve\",\n",
" # Assess agent decision\n",
" grade_documents,\n",
")\n",
"workflow.add_edge(\"generate\", END)\n",
"workflow.add_edge(\"rewrite\", \"agent\")\n",
"\n",
"# Compile\n",
"graph = workflow.compile()"
]
"source": ["from langgraph.graph import END, StateGraph, START\nfrom langgraph.prebuilt import ToolNode\n\n# Define a new graph\nworkflow = StateGraph(AgentState)\n\n# Define the nodes we will cycle between\nworkflow.add_node(\"agent\", agent) # agent\nretrieve = ToolNode([retriever_tool])\nworkflow.add_node(\"retrieve\", retrieve) # retrieval\nworkflow.add_node(\"rewrite\", rewrite) # Re-writing the question\nworkflow.add_node(\n \"generate\", generate\n) # Generating a response after we know the documents are relevant\n# Call agent node to decide to retrieve or not\nworkflow.add_edge(START, \"agent\")\n\n# Decide whether to retrieve\nworkflow.add_conditional_edges(\n \"agent\",\n # Assess agent decision\n tools_condition,\n {\n # Translate the condition outputs to nodes in our graph\n \"tools\": \"retrieve\",\n END: END,\n },\n)\n\n# Edges taken after the `action` node is called.\nworkflow.add_conditional_edges(\n \"retrieve\",\n # Assess agent decision\n grade_documents,\n)\nworkflow.add_edge(\"generate\", END)\nworkflow.add_edge(\"rewrite\", \"agent\")\n\n# Compile\ngraph = workflow.compile()"]
},
{
"cell_type": "code",
@@ -444,15 +169,7 @@
"output_type": "display_data"
}
],
"source": [
"from IPython.display import Image, display\n",
"\n",
"try:\n",
" display(Image(graph.get_graph(xray=True).draw_mermaid_png()))\n",
"except Exception:\n",
" # This requires some extra dependencies and is optional\n",
" pass"
]
"source": ["from IPython.display import Image, display\n\ntry:\n display(Image(graph.get_graph(xray=True).draw_mermaid_png()))\nexcept Exception:\n # This requires some extra dependencies and is optional\n pass"]
},
{
"cell_type": "code",
@@ -486,21 +203,7 @@
]
}
],
"source": [
"import pprint\n",
"\n",
"inputs = {\n",
" \"messages\": [\n",
" (\"user\", \"What does Lilian Weng say about the types of agent memory?\"),\n",
" ]\n",
"}\n",
"for output in graph.stream(inputs):\n",
" for key, value in output.items():\n",
" pprint.pprint(f\"Output from node '{key}':\")\n",
" pprint.pprint(\"---\")\n",
" pprint.pprint(value, indent=2, width=80, depth=None)\n",
" pprint.pprint(\"\\n---\\n\")"
]
"source": ["import pprint\n\ninputs = {\n \"messages\": [\n (\"user\", \"What does Lilian Weng say about the types of agent memory?\"),\n ]\n}\nfor output in graph.stream(inputs):\n for key, value in output.items():\n pprint.pprint(f\"Output from node '{key}':\")\n pprint.pprint(\"---\")\n pprint.pprint(value, indent=2, width=80, depth=None)\n pprint.pprint(\"\\n---\\n\")"]
},
{
"cell_type": "code",
@@ -508,7 +211,7 @@
"id": "189333cc-5d34-4869-9f9b-741210e1096f",
"metadata": {},
"outputs": [],
"source": []
"source": [""]
}
],
"metadata": {
examples/rag/langgraph_crag.ipynb
+15 -370
View File
@@ -47,9 +47,7 @@
"id": "568c84d6-9df6-4b7b-b50d-476c0a64a04b",
"metadata": {},
"outputs": [],
"source": [
"! pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph tavily-python"
]
"source": ["! pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph tavily-python"]
},
{
"cell_type": "markdown",
@@ -65,11 +63,7 @@
"id": "74710419-158d-4270-931c-de83db7b580d",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"OPENAI_API_KEY\"] = \"<your-api-key>\""
]
"source": ["import os\n\nos.environ[\"OPENAI_API_KEY\"] = \"<your-api-key>\""]
},
{
"cell_type": "markdown",
@@ -87,9 +81,7 @@
"id": "c3ac6e65-2d4e-48dd-9fff-40047373332d",
"metadata": {},
"outputs": [],
"source": [
"os.environ[\"TAVILY_API_KEY\"] = \"<your-api-key>\""
]
"source": ["os.environ[\"TAVILY_API_KEY\"] = \"<your-api-key>\""]
},
{
"cell_type": "markdown",
@@ -107,11 +99,7 @@
"id": "e205f57e-5218-478b-ad8e-1723bdb0d45e",
"metadata": {},
"outputs": [],
"source": [
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
"os.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""
]
"source": ["os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""]
},
{
"cell_type": "markdown",
@@ -129,34 +117,7 @@
"id": "3a566a30-cf0e-4330-ad4d-9bf994bdfa86",
"metadata": {},
"outputs": [],
"source": [
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"from langchain_community.document_loaders import WebBaseLoader\n",
"from langchain_community.vectorstores import Chroma\n",
"from langchain_openai import OpenAIEmbeddings\n",
"\n",
"urls = [\n",
" \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
" \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
" \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
"]\n",
"\n",
"docs = [WebBaseLoader(url).load() for url in urls]\n",
"docs_list = [item for sublist in docs for item in sublist]\n",
"\n",
"text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
" chunk_size=250, chunk_overlap=0\n",
")\n",
"doc_splits = text_splitter.split_documents(docs_list)\n",
"\n",
"# Add to vectorDB\n",
"vectorstore = Chroma.from_documents(\n",
" documents=doc_splits,\n",
" collection_name=\"rag-chroma\",\n",
" embedding=OpenAIEmbeddings(),\n",
")\n",
"retriever = vectorstore.as_retriever()"
]
"source": ["from langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_openai import OpenAIEmbeddings\n\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=250, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorDB\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=OpenAIEmbeddings(),\n)\nretriever = vectorstore.as_retriever()"]
},
{
"cell_type": "markdown",
@@ -180,44 +141,7 @@
]
}
],
"source": [
"### Retrieval Grader\n",
"\n",
"from langchain_core.prompts import ChatPromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"\n",
"# Data model\n",
"class GradeDocuments(BaseModel):\n",
" \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
"\n",
" binary_score: str = Field(\n",
" description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
" )\n",
"\n",
"\n",
"# LLM with function call\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
"\n",
"# Prompt\n",
"system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
" If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant. \\n\n",
" Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
"grade_prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\"system\", system),\n",
" (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
" ]\n",
")\n",
"\n",
"retrieval_grader = grade_prompt | structured_llm_grader\n",
"question = \"agent memory\"\n",
"docs = retriever.get_relevant_documents(question)\n",
"doc_txt = docs[1].page_content\n",
"print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
]
"source": ["### Retrieval Grader\n\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\n\n# Data model\nclass GradeDocuments(BaseModel):\n \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n\n binary_score: str = Field(\n description=\"Documents are relevant to the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeDocuments)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\ngrade_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n ]\n)\n\nretrieval_grader = grade_prompt | structured_llm_grader\nquestion = \"agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"]
},
{
"cell_type": "code",
@@ -233,31 +157,7 @@
]
}
],
"source": [
"### Generate\n",
"\n",
"from langchain import hub\n",
"from langchain_core.output_parsers import StrOutputParser\n",
"\n",
"# Prompt\n",
"prompt = hub.pull(\"rlm/rag-prompt\")\n",
"\n",
"# LLM\n",
"llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
"\n",
"\n",
"# Post-processing\n",
"def format_docs(docs):\n",
" return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
"\n",
"\n",
"# Chain\n",
"rag_chain = prompt | llm | StrOutputParser()\n",
"\n",
"# Run\n",
"generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
"print(generation)"
]
"source": ["### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n\n\n# Post-processing\ndef format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"]
},
{
"cell_type": "code",
@@ -276,28 +176,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Question Re-writer\n",
"\n",
"# LLM\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"\n",
"# Prompt\n",
"system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
" for web search. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
"re_write_prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\"system\", system),\n",
" (\n",
" \"human\",\n",
" \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
" ),\n",
" ]\n",
")\n",
"\n",
"question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
"question_rewriter.invoke({\"question\": question})"
]
"source": ["### Question Re-writer\n\n# LLM\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n\n# Prompt\nsystem = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n for web search. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\nre_write_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\n \"human\",\n \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n ),\n ]\n)\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nquestion_rewriter.invoke({\"question\": question})"]
},
{
"cell_type": "markdown",
@@ -313,13 +192,7 @@
"id": "46d51b53-54a9-4e0a-9f14-e39998f5b340",
"metadata": {},
"outputs": [],
"source": [
"### Search\n",
"\n",
"from langchain_community.tools.tavily_search import TavilySearchResults\n",
"\n",
"web_search_tool = TavilySearchResults(k=3)"
]
"source": ["### Search\n\nfrom langchain_community.tools.tavily_search import TavilySearchResults\n\nweb_search_tool = TavilySearchResults(k=3)"]
},
{
"cell_type": "markdown",
@@ -339,28 +212,7 @@
"id": "94b3945f-ef0f-458d-a443-f763903550b0",
"metadata": {},
"outputs": [],
"source": [
"from typing import List\n",
"\n",
"from typing_extensions import TypedDict\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
"\n",
" Attributes:\n",
" question: question\n",
" generation: LLM generation\n",
" web_search: whether to add search\n",
" documents: list of documents\n",
" \"\"\"\n",
"\n",
" question: str\n",
" generation: str\n",
" web_search: str\n",
" documents: List[str]"
]
"source": ["from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n web_search: whether to add search\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n web_search: str\n documents: List[str]"]
},
{
"cell_type": "code",
@@ -368,155 +220,7 @@
"id": "efd639c5-82e2-45e6-a94a-6a4039646ef5",
"metadata": {},
"outputs": [],
"source": [
"from langchain.schema import Document\n",
"\n",
"\n",
"def retrieve(state):\n",
" \"\"\"\n",
" Retrieve documents\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, documents, that contains retrieved documents\n",
" \"\"\"\n",
" print(\"---RETRIEVE---\")\n",
" question = state[\"question\"]\n",
"\n",
" # Retrieval\n",
" documents = retriever.get_relevant_documents(question)\n",
" return {\"documents\": documents, \"question\": question}\n",
"\n",
"\n",
"def generate(state):\n",
" \"\"\"\n",
" Generate answer\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, generation, that contains LLM generation\n",
" \"\"\"\n",
" print(\"---GENERATE---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # RAG generation\n",
" generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
" return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
"\n",
"\n",
"def grade_documents(state):\n",
" \"\"\"\n",
" Determines whether the retrieved documents are relevant to the question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates documents key with only filtered relevant documents\n",
" \"\"\"\n",
"\n",
" print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Score each doc\n",
" filtered_docs = []\n",
" web_search = \"No\"\n",
" for d in documents:\n",
" score = retrieval_grader.invoke(\n",
" {\"question\": question, \"document\": d.page_content}\n",
" )\n",
" grade = score.binary_score\n",
" if grade == \"yes\":\n",
" print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
" filtered_docs.append(d)\n",
" else:\n",
" print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
" web_search = \"Yes\"\n",
" continue\n",
" return {\"documents\": filtered_docs, \"question\": question, \"web_search\": web_search}\n",
"\n",
"\n",
"def transform_query(state):\n",
" \"\"\"\n",
" Transform the query to produce a better question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates question key with a re-phrased question\n",
" \"\"\"\n",
"\n",
" print(\"---TRANSFORM QUERY---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Re-write question\n",
" better_question = question_rewriter.invoke({\"question\": question})\n",
" return {\"documents\": documents, \"question\": better_question}\n",
"\n",
"\n",
"def web_search(state):\n",
" \"\"\"\n",
" Web search based on the re-phrased question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates documents key with appended web results\n",
" \"\"\"\n",
"\n",
" print(\"---WEB SEARCH---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Web search\n",
" docs = web_search_tool.invoke({\"query\": question})\n",
" web_results = \"\\n\".join([d[\"content\"] for d in docs])\n",
" web_results = Document(page_content=web_results)\n",
" documents.append(web_results)\n",
"\n",
" return {\"documents\": documents, \"question\": question}\n",
"\n",
"\n",
"### Edges\n",
"\n",
"\n",
"def decide_to_generate(state):\n",
" \"\"\"\n",
" Determines whether to generate an answer, or re-generate a question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Binary decision for next node to call\n",
" \"\"\"\n",
"\n",
" print(\"---ASSESS GRADED DOCUMENTS---\")\n",
" state[\"question\"]\n",
" web_search = state[\"web_search\"]\n",
" state[\"documents\"]\n",
"\n",
" if web_search == \"Yes\":\n",
" # All documents have been filtered check_relevance\n",
" # We will re-generate a new query\n",
" print(\n",
" \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
" )\n",
" return \"transform_query\"\n",
" else:\n",
" # We have relevant documents, so generate answer\n",
" print(\"---DECISION: GENERATE---\")\n",
" return \"generate\""
]
"source": ["from langchain.schema import Document\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.get_relevant_documents(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n web_search = \"No\"\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n web_search = \"Yes\"\n continue\n return {\"documents\": filtered_docs, \"question\": question, \"web_search\": web_search}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}\n\n\ndef web_search(state):\n \"\"\"\n Web search based on the re-phrased question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with appended web results\n \"\"\"\n\n print(\"---WEB SEARCH---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Web search\n docs = web_search_tool.invoke({\"query\": question})\n web_results = \"\\n\".join([d[\"content\"] for d in docs])\n web_results = Document(page_content=web_results)\n documents.append(web_results)\n\n return {\"documents\": documents, \"question\": question}\n\n\n### Edges\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n web_search = state[\"web_search\"]\n state[\"documents\"]\n\n if web_search == \"Yes\":\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\""]
},
{
"cell_type": "markdown",
@@ -534,36 +238,7 @@
"id": "dedae17a-98c6-474d-90a7-9234b7c8cea0",
"metadata": {},
"outputs": [],
"source": [
"from langgraph.graph import END, StateGraph\n",
"\n",
"workflow = StateGraph(GraphState)\n",
"\n",
"# Define the nodes\n",
"workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
"workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
"workflow.add_node(\"generate\", generate) # generatae\n",
"workflow.add_node(\"transform_query\", transform_query) # transform_query\n",
"workflow.add_node(\"web_search_node\", web_search) # web search\n",
"\n",
"# Build graph\n",
"workflow.set_entry_point(\"retrieve\")\n",
"workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
"workflow.add_conditional_edges(\n",
" \"grade_documents\",\n",
" decide_to_generate,\n",
" {\n",
" \"transform_query\": \"transform_query\",\n",
" \"generate\": \"generate\",\n",
" },\n",
")\n",
"workflow.add_edge(\"transform_query\", \"web_search_node\")\n",
"workflow.add_edge(\"web_search_node\", \"generate\")\n",
"workflow.add_edge(\"generate\", END)\n",
"\n",
"# Compile\n",
"app = workflow.compile()"
]
"source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"transform_query\", transform_query) # transform_query\nworkflow.add_node(\"web_search_node\", web_search) # web search\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"transform_query\": \"transform_query\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"transform_query\", \"web_search_node\")\nworkflow.add_edge(\"web_search_node\", \"generate\")\nworkflow.add_edge(\"generate\", END)\n\n# Compile\napp = workflow.compile()"]
},
{
"cell_type": "code",
@@ -606,22 +281,7 @@
]
}
],
"source": [
"from pprint import pprint\n",
"\n",
"# Run\n",
"inputs = {\"question\": \"What are the types of agent memory?\"}\n",
"for output in app.stream(inputs):\n",
" for key, value in output.items():\n",
" # Node\n",
" pprint(f\"Node '{key}':\")\n",
" # Optional: print full state at each node\n",
" # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
" pprint(\"\\n---\\n\")\n",
"\n",
"# Final generation\n",
"pprint(value[\"generation\"])"
]
"source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"What are the types of agent memory?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
},
{
"cell_type": "code",
@@ -666,22 +326,7 @@
]
}
],
"source": [
"from pprint import pprint\n",
"\n",
"# Run\n",
"inputs = {\"question\": \"How does the AlphaCodium paper work?\"}\n",
"for output in app.stream(inputs):\n",
" for key, value in output.items():\n",
" # Node\n",
" pprint(f\"Node '{key}':\")\n",
" # Optional: print full state at each node\n",
" # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
" pprint(\"\\n---\\n\")\n",
"\n",
"# Final generation\n",
"pprint(value[\"generation\"])"
]
"source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"How does the AlphaCodium paper work?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
},
{
"cell_type": "markdown",
@@ -701,7 +346,7 @@
"id": "6ce65be5-fd12-4ffc-984c-34c132693e69",
"metadata": {},
"outputs": [],
"source": []
"source": [""]
}
],
"metadata": {
examples/rag/langgraph_crag_local.ipynb
+16 -458
View File
@@ -56,10 +56,7 @@
"id": "4a660963-bd3d-4c87-b2e4-b6e432055211",
"metadata": {},
"outputs": [],
"source": [
"%%capture --no-stderr\n",
"%pip install -U langchain_community tiktoken langchainhub scikit-learn langchain langgraph tavily-python nomic[local] langchain-nomic langchain_openai"
]
"source": ["%%capture --no-stderr\n%pip install -U langchain_community tiktoken langchainhub scikit-learn langchain langgraph tavily-python nomic[local] langchain-nomic langchain_openai"]
},
{
"cell_type": "code",
@@ -67,11 +64,7 @@
"id": "68316ba0-854b-41e1-9af5-1f9e965946e3",
"metadata": {},
"outputs": [],
"source": [
"# Search\n",
"import os\n",
"os.environ[\"TAVILY_API_KEY\"] = \"xxx\""
]
"source": ["# Search\nimport os\nos.environ[\"TAVILY_API_KEY\"] = \"xxx\""]
},
{
"cell_type": "code",
@@ -79,10 +72,7 @@
"id": "0be68860-dded-481e-9fc7-a5042bf92c04",
"metadata": {},
"outputs": [],
"source": [
"# Embedding (optional)\n",
"os.environ[\"OPENAI_API_KEY\"] = \"xxx\""
]
"source": ["# Embedding (optional)\nos.environ[\"OPENAI_API_KEY\"] = \"xxx\""]
},
{
"cell_type": "code",
@@ -90,13 +80,7 @@
"id": "7248ab88-2b97-41eb-8dbb-4ea65525ed9a",
"metadata": {},
"outputs": [],
"source": [
"# Tracing and testing (optional)\n",
"os.environ[\"LANGCHAIN_API_KEY\"] = \"xxx\"\n",
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
"os.environ[\"LANGCHAIN_PROJECT\"] = \"corrective-rag-agent-testing\""
]
"source": ["# Tracing and testing (optional)\nos.environ[\"LANGCHAIN_API_KEY\"] = \"xxx\"\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_PROJECT\"] = \"corrective-rag-agent-testing\""]
},
{
"cell_type": "markdown",
@@ -114,11 +98,7 @@
"id": "2f4db331-c4d0-4c7c-a9a5-0bebc8a89c6c",
"metadata": {},
"outputs": [],
"source": [
"local_llm = \"llama3\"\n",
"model_tested = \"llama3-8b\"\n",
"metadata = f\"CRAG, {model_tested}\""
]
"source": ["local_llm = \"llama3\"\nmodel_tested = \"llama3-8b\"\nmetadata = f\"CRAG, {model_tested}\""]
},
{
"cell_type": "markdown",
@@ -144,48 +124,7 @@
]
}
],
"source": [
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"from langchain_community.document_loaders import WebBaseLoader\n",
"from langchain_community.vectorstores import SKLearnVectorStore\n",
"from langchain_nomic.embeddings import NomicEmbeddings # local\n",
"from langchain_openai import OpenAIEmbeddings # api\n",
"\n",
"# List of URLs to load documents from\n",
"urls = [\n",
" \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
" \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
" \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
"]\n",
"\n",
"# Load documents from the URLs\n",
"docs = [WebBaseLoader(url).load() for url in urls]\n",
"docs_list = [item for sublist in docs for item in sublist]\n",
"\n",
"# Initialize a text splitter with specified chunk size and overlap\n",
"text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
" chunk_size=250, chunk_overlap=0\n",
")\n",
"\n",
"# Split the documents into chunks\n",
"doc_splits = text_splitter.split_documents(docs_list)\n",
"\n",
"# Embedding\n",
"'''\n",
"embedding=NomicEmbeddings(\n",
" model=\"nomic-embed-text-v1.5\",\n",
" inference_mode=\"local\",\n",
")\n",
"'''\n",
"embedding = OpenAIEmbeddings()\n",
"\n",
"# Add the document chunks to the \"vector store\"\n",
"vectorstore = SKLearnVectorStore.from_documents(\n",
" documents=doc_splits,\n",
" embedding=embedding,\n",
")\n",
"retriever = vectorstore.as_retriever(k=4)"
]
"source": ["from langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import SKLearnVectorStore\nfrom langchain_nomic.embeddings import NomicEmbeddings # local\nfrom langchain_openai import OpenAIEmbeddings # api\n\n# List of URLs to load documents from\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\n# Load documents from the URLs\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\n# Initialize a text splitter with specified chunk size and overlap\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=250, chunk_overlap=0\n)\n\n# Split the documents into chunks\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Embedding\n'''\nembedding=NomicEmbeddings(\n model=\"nomic-embed-text-v1.5\",\n inference_mode=\"local\",\n)\n'''\nembedding = OpenAIEmbeddings()\n\n# Add the document chunks to the \"vector store\"\nvectorstore = SKLearnVectorStore.from_documents(\n documents=doc_splits,\n embedding=embedding,\n)\nretriever = vectorstore.as_retriever(k=4)"]
},
{
"attachments": {},
@@ -210,47 +149,7 @@
]
}
],
"source": [
"### Retrieval Grader\n",
"\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain_community.chat_models import ChatOllama\n",
"from langchain_core.output_parsers import JsonOutputParser\n",
"from langchain_mistralai.chat_models import ChatMistralAI\n",
"\n",
"# LLM\n",
"llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
"\n",
"# Prompt\n",
"prompt = PromptTemplate(\n",
" template=\"\"\"You are a teacher grading a quiz. You will be given: \n",
" 1/ a QUESTION\n",
" 2/ A FACT provided by the student\n",
" \n",
" You are grading RELEVANCE RECALL:\n",
" A score of 1 means that ANY of the statements in the FACT are relevant to the QUESTION. \n",
" A score of 0 means that NONE of the statements in the FACT are relevant to the QUESTION. \n",
" 1 is the highest (best) score. 0 is the lowest score you can give. \n",
" \n",
" Explain your reasoning in a step-by-step manner. Ensure your reasoning and conclusion are correct. \n",
" \n",
" Avoid simply stating the correct answer at the outset.\n",
" \n",
" Question: {question} \\n\n",
" Fact: \\n\\n {documents} \\n\\n\n",
" \n",
" Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n",
" Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\n",
" \"\"\",\n",
" input_variables=[\"question\", \"documents\"],\n",
")\n",
"\n",
"retrieval_grader = prompt | llm | JsonOutputParser()\n",
"question = \"agent memory\"\n",
"docs = retriever.invoke(question)\n",
"doc_txt = docs[1].page_content\n",
"print(retrieval_grader.invoke({\"question\": question, \"documents\": doc_txt}))"
]
"source": ["### Retrieval Grader\n\nfrom langchain.prompts import PromptTemplate\nfrom langchain_community.chat_models import ChatOllama\nfrom langchain_core.output_parsers import JsonOutputParser\nfrom langchain_mistralai.chat_models import ChatMistralAI\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are a teacher grading a quiz. You will be given: \n 1/ a QUESTION\n 2/ A FACT provided by the student\n \n You are grading RELEVANCE RECALL:\n A score of 1 means that ANY of the statements in the FACT are relevant to the QUESTION. \n A score of 0 means that NONE of the statements in the FACT are relevant to the QUESTION. \n 1 is the highest (best) score. 0 is the lowest score you can give. \n \n Explain your reasoning in a step-by-step manner. Ensure your reasoning and conclusion are correct. \n \n Avoid simply stating the correct answer at the outset.\n \n Question: {question} \\n\n Fact: \\n\\n {documents} \\n\\n\n \n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\n \"\"\",\n input_variables=[\"question\", \"documents\"],\n)\n\nretrieval_grader = prompt | llm | JsonOutputParser()\nquestion = \"agent memory\"\ndocs = retriever.invoke(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"documents\": doc_txt}))"]
},
{
"cell_type": "code",
@@ -266,37 +165,7 @@
]
}
],
"source": [
"### Generate\n",
"\n",
"from langchain_core.output_parsers import StrOutputParser\n",
"\n",
"# Prompt\n",
"prompt = PromptTemplate(\n",
" template=\"\"\"You are an assistant for question-answering tasks. \n",
" \n",
" Use the following documents to answer the question. \n",
" \n",
" If you don't know the answer, just say that you don't know. \n",
" \n",
" Use three sentences maximum and keep the answer concise:\n",
" Question: {question} \n",
" Documents: {documents} \n",
" Answer: \n",
" \"\"\",\n",
" input_variables=[\"question\", \"documents\"],\n",
")\n",
"\n",
"# LLM\n",
"llm = ChatOllama(model=local_llm, temperature=0)\n",
"\n",
"# Chain\n",
"rag_chain = prompt | llm | StrOutputParser()\n",
"\n",
"# Run\n",
"generation = rag_chain.invoke({\"documents\": docs, \"question\": question})\n",
"print(generation)"
]
"source": ["### Generate\n\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are an assistant for question-answering tasks. \n \n Use the following documents to answer the question. \n \n If you don't know the answer, just say that you don't know. \n \n Use three sentences maximum and keep the answer concise:\n Question: {question} \n Documents: {documents} \n Answer: \n \"\"\",\n input_variables=[\"question\", \"documents\"],\n)\n\n# LLM\nllm = ChatOllama(model=local_llm, temperature=0)\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"documents\": docs, \"question\": question})\nprint(generation)"]
},
{
"cell_type": "code",
@@ -304,13 +173,7 @@
"id": "b36a2f36-bc5f-408d-a5e8-3fa203c233f6",
"metadata": {},
"outputs": [],
"source": [
"### Search\n",
"\n",
"from langchain_community.tools.tavily_search import TavilySearchResults\n",
"\n",
"web_search_tool = TavilySearchResults(k=3)"
]
"source": ["### Search\n\nfrom langchain_community.tools.tavily_search import TavilySearchResults\n\nweb_search_tool = TavilySearchResults(k=3)"]
},
{
"cell_type": "markdown",
@@ -339,177 +202,7 @@
"output_type": "display_data"
}
],
"source": [
"from typing import List\n",
"from typing_extensions import TypedDict\n",
"from IPython.display import Image, display\n",
"from langchain.schema import Document\n",
"from langgraph.graph import START, END, StateGraph\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
"\n",
" Attributes:\n",
" question: question\n",
" generation: LLM generation\n",
" search: whether to add search\n",
" documents: list of documents\n",
" \"\"\"\n",
"\n",
" question: str\n",
" generation: str\n",
" search: str\n",
" documents: List[str]\n",
" steps: List[str]\n",
"\n",
"\n",
"def retrieve(state):\n",
" \"\"\"\n",
" Retrieve documents\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, documents, that contains retrieved documents\n",
" \"\"\"\n",
" question = state[\"question\"]\n",
" documents = retriever.invoke(question)\n",
" steps = state[\"steps\"]\n",
" steps.append(\"retrieve_documents\")\n",
" return {\"documents\": documents, \"question\": question, \"steps\": steps}\n",
"\n",
"\n",
"def generate(state):\n",
" \"\"\"\n",
" Generate answer\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, generation, that contains LLM generation\n",
" \"\"\"\n",
"\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
" generation = rag_chain.invoke({\"documents\": documents, \"question\": question})\n",
" steps = state[\"steps\"]\n",
" steps.append(\"generate_answer\")\n",
" return {\n",
" \"documents\": documents,\n",
" \"question\": question,\n",
" \"generation\": generation,\n",
" \"steps\": steps,\n",
" }\n",
"\n",
"\n",
"def grade_documents(state):\n",
" \"\"\"\n",
" Determines whether the retrieved documents are relevant to the question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates documents key with only filtered relevant documents\n",
" \"\"\"\n",
"\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
" steps = state[\"steps\"]\n",
" steps.append(\"grade_document_retrieval\")\n",
" filtered_docs = []\n",
" search = \"No\"\n",
" for d in documents:\n",
" score = retrieval_grader.invoke(\n",
" {\"question\": question, \"documents\": d.page_content}\n",
" )\n",
" grade = score[\"score\"]\n",
" if grade == \"yes\":\n",
" filtered_docs.append(d)\n",
" else:\n",
" search = \"Yes\"\n",
" continue\n",
" return {\n",
" \"documents\": filtered_docs,\n",
" \"question\": question,\n",
" \"search\": search,\n",
" \"steps\": steps,\n",
" }\n",
"\n",
"\n",
"def web_search(state):\n",
" \"\"\"\n",
" Web search based on the re-phrased question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates documents key with appended web results\n",
" \"\"\"\n",
"\n",
" question = state[\"question\"]\n",
" documents = state.get(\"documents\", [])\n",
" steps = state[\"steps\"]\n",
" steps.append(\"web_search\")\n",
" web_results = web_search_tool.invoke({\"query\": question})\n",
" documents.extend(\n",
" [\n",
" Document(page_content=d[\"content\"], metadata={\"url\": d[\"url\"]})\n",
" for d in web_results\n",
" ]\n",
" )\n",
" return {\"documents\": documents, \"question\": question, \"steps\": steps}\n",
"\n",
"\n",
"def decide_to_generate(state):\n",
" \"\"\"\n",
" Determines whether to generate an answer, or re-generate a question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Binary decision for next node to call\n",
" \"\"\"\n",
" search = state[\"search\"]\n",
" if search == \"Yes\":\n",
" return \"search\"\n",
" else:\n",
" return \"generate\"\n",
"\n",
"\n",
"# Graph\n",
"workflow = StateGraph(GraphState)\n",
"\n",
"# Define the nodes\n",
"workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
"workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
"workflow.add_node(\"generate\", generate) # generatae\n",
"workflow.add_node(\"web_search\", web_search) # web search\n",
"\n",
"# Build graph\n",
"workflow.set_entry_point(\"retrieve\")\n",
"workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
"workflow.add_conditional_edges(\n",
" \"grade_documents\",\n",
" decide_to_generate,\n",
" {\n",
" \"search\": \"web_search\",\n",
" \"generate\": \"generate\",\n",
" },\n",
")\n",
"workflow.add_edge(\"web_search\", \"generate\")\n",
"workflow.add_edge(\"generate\", END)\n",
"\n",
"custom_graph = workflow.compile()\n",
"\n",
"display(Image(custom_graph.get_graph(xray=True).draw_mermaid_png()))"
]
"source": ["from typing import List\nfrom typing_extensions import TypedDict\nfrom IPython.display import Image, display\nfrom langchain.schema import Document\nfrom langgraph.graph import START, END, StateGraph\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n search: whether to add search\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n search: str\n documents: List[str]\n steps: List[str]\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n question = state[\"question\"]\n documents = retriever.invoke(question)\n steps = state[\"steps\"]\n steps.append(\"retrieve_documents\")\n return {\"documents\": documents, \"question\": question, \"steps\": steps}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = rag_chain.invoke({\"documents\": documents, \"question\": question})\n steps = state[\"steps\"]\n steps.append(\"generate_answer\")\n return {\n \"documents\": documents,\n \"question\": question,\n \"generation\": generation,\n \"steps\": steps,\n }\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n question = state[\"question\"]\n documents = state[\"documents\"]\n steps = state[\"steps\"]\n steps.append(\"grade_document_retrieval\")\n filtered_docs = []\n search = \"No\"\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"documents\": d.page_content}\n )\n grade = score[\"score\"]\n if grade == \"yes\":\n filtered_docs.append(d)\n else:\n search = \"Yes\"\n continue\n return {\n \"documents\": filtered_docs,\n \"question\": question,\n \"search\": search,\n \"steps\": steps,\n }\n\n\ndef web_search(state):\n \"\"\"\n Web search based on the re-phrased question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with appended web results\n \"\"\"\n\n question = state[\"question\"]\n documents = state.get(\"documents\", [])\n steps = state[\"steps\"]\n steps.append(\"web_search\")\n web_results = web_search_tool.invoke({\"query\": question})\n documents.extend(\n [\n Document(page_content=d[\"content\"], metadata={\"url\": d[\"url\"]})\n for d in web_results\n ]\n )\n return {\"documents\": documents, \"question\": question, \"steps\": steps}\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n search = state[\"search\"]\n if search == \"Yes\":\n return \"search\"\n else:\n return \"generate\"\n\n\n# Graph\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"web_search\", web_search) # web search\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"search\": \"web_search\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"web_search\", \"generate\")\nworkflow.add_edge(\"generate\", END)\n\ncustom_graph = workflow.compile()\n\ndisplay(Image(custom_graph.get_graph(xray=True).draw_mermaid_png()))"]
},
{
"cell_type": "code",
@@ -532,21 +225,7 @@
"output_type": "execute_result"
}
],
"source": [
"import uuid\n",
"\n",
"def predict_custom_agent_local_answer(example: dict):\n",
" config = {\"configurable\": {\"thread_id\": str(uuid.uuid4())}}\n",
" state_dict = custom_graph.invoke(\n",
" {\"question\": example[\"input\"], \"steps\": []}, config\n",
" )\n",
" return {\"response\": state_dict[\"generation\"], \"steps\": state_dict[\"steps\"]}\n",
"\n",
"\n",
"example = {\"input\": \"What are the types of agent memory?\"}\n",
"response = predict_custom_agent_local_answer(example)\n",
"response"
]
"source": ["import uuid\n\ndef predict_custom_agent_local_answer(example: dict):\n config = {\"configurable\": {\"thread_id\": str(uuid.uuid4())}}\n state_dict = custom_graph.invoke(\n {\"question\": example[\"input\"], \"steps\": []}, config\n )\n return {\"response\": state_dict[\"generation\"], \"steps\": state_dict[\"steps\"]}\n\n\nexample = {\"input\": \"What are the types of agent memory?\"}\nresponse = predict_custom_agent_local_answer(example)\nresponse"]
},
{
"cell_type": "markdown",
@@ -582,41 +261,7 @@
"id": "b83706ac-724b-46b1-9f08-66e6c4fac742",
"metadata": {},
"outputs": [],
"source": [
"from langsmith import Client\n",
"\n",
"client = Client()\n",
"\n",
"# Create a dataset\n",
"examples = [\n",
" (\n",
" \"How does the ReAct agent use self-reflection? \",\n",
" \"ReAct integrates reasoning and acting, performing actions - such tools like Wikipedia search API - and then observing / reasoning about the tool outputs.\",\n",
" ),\n",
" (\n",
" \"What are the types of biases that can arise with few-shot prompting?\",\n",
" \"The biases that can arise with few-shot prompting include (1) Majority label bias, (2) Recency bias, and (3) Common token bias.\",\n",
" ),\n",
" (\n",
" \"What are five types of adversarial attacks?\",\n",
" \"Five types of adversarial attacks are (1) Token manipulation, (2) Gradient based attack, (3) Jailbreak prompting, (4) Human red-teaming, (5) Model red-teaming.\",\n",
" ),\n",
" (\n",
" \"Who did the Chicago Bears draft first in the 2024 NFL draft”?\",\n",
" \"The Chicago Bears drafted Caleb Williams first in the 2024 NFL draft.\",\n",
" ),\n",
" (\"Who won the 2024 NBA finals?\", \"The Boston Celtics on the 2024 NBA finals\"),\n",
"]\n",
"\n",
"# Save it\n",
"dataset_name = \"Corrective RAG Agent Testing\"\n",
"if not client.has_dataset(dataset_name=dataset_name):\n",
" dataset = client.create_dataset(dataset_name=dataset_name)\n",
" inputs, outputs = zip(\n",
" *[({\"input\": text}, {\"output\": label}) for text, label in examples]\n",
" )\n",
" client.create_examples(inputs=inputs, outputs=outputs, dataset_id=dataset.id)"
]
"source": ["from langsmith import Client\n\nclient = Client()\n\n# Create a dataset\nexamples = [\n (\n \"How does the ReAct agent use self-reflection? \",\n \"ReAct integrates reasoning and acting, performing actions - such tools like Wikipedia search API - and then observing / reasoning about the tool outputs.\",\n ),\n (\n \"What are the types of biases that can arise with few-shot prompting?\",\n \"The biases that can arise with few-shot prompting include (1) Majority label bias, (2) Recency bias, and (3) Common token bias.\",\n ),\n (\n \"What are five types of adversarial attacks?\",\n \"Five types of adversarial attacks are (1) Token manipulation, (2) Gradient based attack, (3) Jailbreak prompting, (4) Human red-teaming, (5) Model red-teaming.\",\n ),\n (\n \"Who did the Chicago Bears draft first in the 2024 NFL draft”?\",\n \"The Chicago Bears drafted Caleb Williams first in the 2024 NFL draft.\",\n ),\n (\"Who won the 2024 NBA finals?\", \"The Boston Celtics on the 2024 NBA finals\"),\n]\n\n# Save it\ndataset_name = \"Corrective RAG Agent Testing\"\nif not client.has_dataset(dataset_name=dataset_name):\n dataset = client.create_dataset(dataset_name=dataset_name)\n inputs, outputs = zip(\n *[({\"input\": text}, {\"output\": label}) for text, label in examples]\n )\n client.create_examples(inputs=inputs, outputs=outputs, dataset_id=dataset.id)"]
},
{
"cell_type": "markdown",
@@ -636,38 +281,7 @@
"id": "0a63776c-f9cd-46ce-b8cf-95c066dc5b06",
"metadata": {},
"outputs": [],
"source": [
"from langchain import hub\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"# Grade prompt\n",
"grade_prompt_answer_accuracy = hub.pull(\"langchain-ai/rag-answer-vs-reference\")\n",
"\n",
"def answer_evaluator(run, example) -> dict:\n",
" \"\"\"\n",
" A simple evaluator for RAG answer accuracy\n",
" \"\"\"\n",
"\n",
" # Get the question, the ground truth reference answer, RAG chain answer prediction\n",
" input_question = example.inputs[\"input\"]\n",
" reference = example.outputs[\"output\"]\n",
" prediction = run.outputs[\"response\"]\n",
"\n",
" # Define an LLM grader\n",
" llm = ChatOpenAI(model=\"gpt-4o\", temperature=0)\n",
" answer_grader = grade_prompt_answer_accuracy | llm\n",
"\n",
" # Run evaluator\n",
" score = answer_grader.invoke(\n",
" {\n",
" \"question\": input_question,\n",
" \"correct_answer\": reference,\n",
" \"student_answer\": prediction,\n",
" }\n",
" )\n",
" score = score[\"Score\"]\n",
" return {\"key\": \"answer_v_reference_score\", \"score\": score}"
]
"source": ["from langchain import hub\nfrom langchain_openai import ChatOpenAI\n\n# Grade prompt\ngrade_prompt_answer_accuracy = hub.pull(\"langchain-ai/rag-answer-vs-reference\")\n\ndef answer_evaluator(run, example) -> dict:\n \"\"\"\n A simple evaluator for RAG answer accuracy\n \"\"\"\n\n # Get the question, the ground truth reference answer, RAG chain answer prediction\n input_question = example.inputs[\"input\"]\n reference = example.outputs[\"output\"]\n prediction = run.outputs[\"response\"]\n\n # Define an LLM grader\n llm = ChatOpenAI(model=\"gpt-4o\", temperature=0)\n answer_grader = grade_prompt_answer_accuracy | llm\n\n # Run evaluator\n score = answer_grader.invoke(\n {\n \"question\": input_question,\n \"correct_answer\": reference,\n \"student_answer\": prediction,\n }\n )\n score = score[\"Score\"]\n return {\"key\": \"answer_v_reference_score\", \"score\": score}"]
},
{
"cell_type": "markdown",
@@ -687,50 +301,7 @@
"id": "deb28175-27a1-4afc-9747-2983e87fc881",
"metadata": {},
"outputs": [],
"source": [
"from langsmith.schemas import Example, Run\n",
"\n",
"# Reasoning traces that we expect the agents to take\n",
"expected_trajectory_1 = [\n",
" \"retrieve_documents\",\n",
" \"grade_document_retrieval\",\n",
" \"web_search\",\n",
" \"generate_answer\",\n",
"]\n",
"expected_trajectory_2 = [\n",
" \"retrieve_documents\",\n",
" \"grade_document_retrieval\",\n",
" \"generate_answer\",\n",
"]\n",
"\n",
"def check_trajectory_react(root_run: Run, example: Example) -> dict:\n",
" \"\"\"\n",
" Check if all expected tools are called in exact order and without any additional tool calls.\n",
" \"\"\"\n",
" messages = root_run.outputs[\"messages\"]\n",
" tool_calls = find_tool_calls_react(messages)\n",
" print(f\"Tool calls ReAct agent: {tool_calls}\")\n",
" if tool_calls == expected_trajectory_1 or tool_calls == expected_trajectory_2:\n",
" score = 1\n",
" else:\n",
" score = 0\n",
"\n",
" return {\"score\": int(score), \"key\": \"tool_calls_in_exact_order\"}\n",
"\n",
"\n",
"def check_trajectory_custom(root_run: Run, example: Example) -> dict:\n",
" \"\"\"\n",
" Check if all expected tools are called in exact order and without any additional tool calls.\n",
" \"\"\"\n",
" tool_calls = root_run.outputs[\"steps\"]\n",
" print(f\"Tool calls custom agent: {tool_calls}\")\n",
" if tool_calls == expected_trajectory_1 or tool_calls == expected_trajectory_2:\n",
" score = 1\n",
" else:\n",
" score = 0\n",
"\n",
" return {\"score\": int(score), \"key\": \"tool_calls_in_exact_order\"}"
]
"source": ["from langsmith.schemas import Example, Run\n\n# Reasoning traces that we expect the agents to take\nexpected_trajectory_1 = [\n \"retrieve_documents\",\n \"grade_document_retrieval\",\n \"web_search\",\n \"generate_answer\",\n]\nexpected_trajectory_2 = [\n \"retrieve_documents\",\n \"grade_document_retrieval\",\n \"generate_answer\",\n]\n\ndef check_trajectory_react(root_run: Run, example: Example) -> dict:\n \"\"\"\n Check if all expected tools are called in exact order and without any additional tool calls.\n \"\"\"\n messages = root_run.outputs[\"messages\"]\n tool_calls = find_tool_calls_react(messages)\n print(f\"Tool calls ReAct agent: {tool_calls}\")\n if tool_calls == expected_trajectory_1 or tool_calls == expected_trajectory_2:\n score = 1\n else:\n score = 0\n\n return {\"score\": int(score), \"key\": \"tool_calls_in_exact_order\"}\n\n\ndef check_trajectory_custom(root_run: Run, example: Example) -> dict:\n \"\"\"\n Check if all expected tools are called in exact order and without any additional tool calls.\n \"\"\"\n tool_calls = root_run.outputs[\"steps\"]\n print(f\"Tool calls custom agent: {tool_calls}\")\n if tool_calls == expected_trajectory_1 or tool_calls == expected_trajectory_2:\n score = 1\n else:\n score = 0\n\n return {\"score\": int(score), \"key\": \"tool_calls_in_exact_order\"}"]
},
{
"cell_type": "code",
@@ -784,20 +355,7 @@
]
}
],
"source": [
"from langsmith.evaluation import evaluate\n",
"\n",
"experiment_prefix = f\"custom-agent-{model_tested}\"\n",
"experiment_results = evaluate(\n",
" predict_custom_agent_local_answer,\n",
" data=dataset_name,\n",
" evaluators=[answer_evaluator, check_trajectory_custom],\n",
" experiment_prefix=experiment_prefix + \"-answer-and-tool-use\",\n",
" num_repetitions=3,\n",
" max_concurrency=1, # Use when running locally\n",
" metadata={\"version\": metadata},\n",
")"
]
"source": ["from langsmith.evaluation import evaluate\n\nexperiment_prefix = f\"custom-agent-{model_tested}\"\nexperiment_results = evaluate(\n predict_custom_agent_local_answer,\n data=dataset_name,\n evaluators=[answer_evaluator, check_trajectory_custom],\n experiment_prefix=experiment_prefix + \"-answer-and-tool-use\",\n num_repetitions=3,\n max_concurrency=1, # Use when running locally\n metadata={\"version\": metadata},\n)"]
},
{
"attachments": {
@@ -824,7 +382,7 @@
"id": "79295798-0181-417e-abad-11dddb6ff05e",
"metadata": {},
"outputs": [],
"source": []
"source": [""]
}
],
"metadata": {
examples/rag/langgraph_rag_agent_llama3_local.ipynb
+15 -480
View File
File diff suppressed because one or more lines are too long
examples/rag/langgraph_self_rag.ipynb
+15 -435
View File
@@ -59,9 +59,7 @@
"id": "a384cc48-0425-4e8f-aafc-cfb8e56025c9",
"metadata": {},
"outputs": [],
"source": [
"! pip install -U langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph"
]
"source": ["! pip install -U langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph"]
},
{
"cell_type": "markdown",
@@ -77,11 +75,7 @@
"id": "f18b63c7-d0d3-41c1-ae6b-5a0f1b8ccf0f",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"OPENAI_API_KEY\"] = \"<your-api-key>\""
]
"source": ["import os\n\nos.environ[\"OPENAI_API_KEY\"] = \"<your-api-key>\""]
},
{
"cell_type": "markdown",
@@ -99,11 +93,7 @@
"id": "ccc3dae5-1df6-48ca-af8a-50f0e6128876",
"metadata": {},
"outputs": [],
"source": [
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
"os.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""
]
"source": ["os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""]
},
{
"cell_type": "markdown",
@@ -121,34 +111,7 @@
"id": "565a6d44-2c9f-4fff-b1ec-eea05df9350d",
"metadata": {},
"outputs": [],
"source": [
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"from langchain_community.document_loaders import WebBaseLoader\n",
"from langchain_community.vectorstores import Chroma\n",
"from langchain_openai import OpenAIEmbeddings\n",
"\n",
"urls = [\n",
" \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
" \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
" \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
"]\n",
"\n",
"docs = [WebBaseLoader(url).load() for url in urls]\n",
"docs_list = [item for sublist in docs for item in sublist]\n",
"\n",
"text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
" chunk_size=250, chunk_overlap=0\n",
")\n",
"doc_splits = text_splitter.split_documents(docs_list)\n",
"\n",
"# Add to vectorDB\n",
"vectorstore = Chroma.from_documents(\n",
" documents=doc_splits,\n",
" collection_name=\"rag-chroma\",\n",
" embedding=OpenAIEmbeddings(),\n",
")\n",
"retriever = vectorstore.as_retriever()"
]
"source": ["from langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_openai import OpenAIEmbeddings\n\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=250, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorDB\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=OpenAIEmbeddings(),\n)\nretriever = vectorstore.as_retriever()"]
},
{
"cell_type": "markdown",
@@ -180,46 +143,7 @@
]
}
],
"source": [
"### Retrieval Grader\n",
"\n",
"\n",
"from langchain_core.prompts import ChatPromptTemplate\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"\n",
"# Data model\n",
"class GradeDocuments(BaseModel):\n",
" \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
"\n",
" binary_score: str = Field(\n",
" description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
" )\n",
"\n",
"\n",
"# LLM with function call\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
"\n",
"# Prompt\n",
"system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
" It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
" If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
" Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
"grade_prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\"system\", system),\n",
" (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
" ]\n",
")\n",
"\n",
"retrieval_grader = grade_prompt | structured_llm_grader\n",
"question = \"agent memory\"\n",
"docs = retriever.get_relevant_documents(question)\n",
"doc_txt = docs[1].page_content\n",
"print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
]
"source": ["### Retrieval Grader\n\n\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\n\n# Data model\nclass GradeDocuments(BaseModel):\n \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n\n binary_score: str = Field(\n description=\"Documents are relevant to the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeDocuments)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\ngrade_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n ]\n)\n\nretrieval_grader = grade_prompt | structured_llm_grader\nquestion = \"agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"]
},
{
"cell_type": "code",
@@ -235,31 +159,7 @@
]
}
],
"source": [
"### Generate\n",
"\n",
"from langchain import hub\n",
"from langchain_core.output_parsers import StrOutputParser\n",
"\n",
"# Prompt\n",
"prompt = hub.pull(\"rlm/rag-prompt\")\n",
"\n",
"# LLM\n",
"llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
"\n",
"\n",
"# Post-processing\n",
"def format_docs(docs):\n",
" return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
"\n",
"\n",
"# Chain\n",
"rag_chain = prompt | llm | StrOutputParser()\n",
"\n",
"# Run\n",
"generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
"print(generation)"
]
"source": ["### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n\n\n# Post-processing\ndef format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"]
},
{
"cell_type": "code",
@@ -278,36 +178,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Hallucination Grader\n",
"\n",
"\n",
"# Data model\n",
"class GradeHallucinations(BaseModel):\n",
" \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n",
"\n",
" binary_score: str = Field(\n",
" description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n",
" )\n",
"\n",
"\n",
"# LLM with function call\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"structured_llm_grader = llm.with_structured_output(GradeHallucinations)\n",
"\n",
"# Prompt\n",
"system = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n \n",
" Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\n",
"hallucination_prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\"system\", system),\n",
" (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n",
" ]\n",
")\n",
"\n",
"hallucination_grader = hallucination_prompt | structured_llm_grader\n",
"hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
]
"source": ["### Hallucination Grader\n\n\n# Data model\nclass GradeHallucinations(BaseModel):\n \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n\n binary_score: str = Field(\n description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeHallucinations)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n \n Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\nhallucination_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n ]\n)\n\nhallucination_grader = hallucination_prompt | structured_llm_grader\nhallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"]
},
{
"cell_type": "code",
@@ -326,36 +197,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Answer Grader\n",
"\n",
"\n",
"# Data model\n",
"class GradeAnswer(BaseModel):\n",
" \"\"\"Binary score to assess answer addresses question.\"\"\"\n",
"\n",
" binary_score: str = Field(\n",
" description=\"Answer addresses the question, 'yes' or 'no'\"\n",
" )\n",
"\n",
"\n",
"# LLM with function call\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"structured_llm_grader = llm.with_structured_output(GradeAnswer)\n",
"\n",
"# Prompt\n",
"system = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n \n",
" Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\n",
"answer_prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\"system\", system),\n",
" (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n",
" ]\n",
")\n",
"\n",
"answer_grader = answer_prompt | structured_llm_grader\n",
"answer_grader.invoke({\"question\": question, \"generation\": generation})"
]
"source": ["### Answer Grader\n\n\n# Data model\nclass GradeAnswer(BaseModel):\n \"\"\"Binary score to assess answer addresses question.\"\"\"\n\n binary_score: str = Field(\n description=\"Answer addresses the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeAnswer)\n\n# Prompt\nsystem = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n \n Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\nanswer_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n ]\n)\n\nanswer_grader = answer_prompt | structured_llm_grader\nanswer_grader.invoke({\"question\": question, \"generation\": generation})"]
},
{
"cell_type": "code",
@@ -374,28 +216,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Question Re-writer\n",
"\n",
"# LLM\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"\n",
"# Prompt\n",
"system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
" for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
"re_write_prompt = ChatPromptTemplate.from_messages(\n",
" [\n",
" (\"system\", system),\n",
" (\n",
" \"human\",\n",
" \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
" ),\n",
" ]\n",
")\n",
"\n",
"question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
"question_rewriter.invoke({\"question\": question})"
]
"source": ["### Question Re-writer\n\n# LLM\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n\n# Prompt\nsystem = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\nre_write_prompt = ChatPromptTemplate.from_messages(\n [\n (\"system\", system),\n (\n \"human\",\n \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n ),\n ]\n)\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nquestion_rewriter.invoke({\"question\": question})"]
},
{
"cell_type": "markdown",
@@ -415,26 +236,7 @@
"id": "f1617e9e-66a8-4c1a-a1fe-cc936284c085",
"metadata": {},
"outputs": [],
"source": [
"from typing import List\n",
"\n",
"from typing_extensions import TypedDict\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
"\n",
" Attributes:\n",
" question: question\n",
" generation: LLM generation\n",
" documents: list of documents\n",
" \"\"\"\n",
"\n",
" question: str\n",
" generation: str\n",
" documents: List[str]"
]
"source": ["from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n documents: List[str]"]
},
{
"cell_type": "code",
@@ -442,167 +244,7 @@
"id": "add509d8-6682-4127-8d95-13dd37d79702",
"metadata": {},
"outputs": [],
"source": [
"### Nodes\n",
"\n",
"\n",
"def retrieve(state):\n",
" \"\"\"\n",
" Retrieve documents\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, documents, that contains retrieved documents\n",
" \"\"\"\n",
" print(\"---RETRIEVE---\")\n",
" question = state[\"question\"]\n",
"\n",
" # Retrieval\n",
" documents = retriever.get_relevant_documents(question)\n",
" return {\"documents\": documents, \"question\": question}\n",
"\n",
"\n",
"def generate(state):\n",
" \"\"\"\n",
" Generate answer\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, generation, that contains LLM generation\n",
" \"\"\"\n",
" print(\"---GENERATE---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # RAG generation\n",
" generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
" return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
"\n",
"\n",
"def grade_documents(state):\n",
" \"\"\"\n",
" Determines whether the retrieved documents are relevant to the question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates documents key with only filtered relevant documents\n",
" \"\"\"\n",
"\n",
" print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Score each doc\n",
" filtered_docs = []\n",
" for d in documents:\n",
" score = retrieval_grader.invoke(\n",
" {\"question\": question, \"document\": d.page_content}\n",
" )\n",
" grade = score.binary_score\n",
" if grade == \"yes\":\n",
" print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
" filtered_docs.append(d)\n",
" else:\n",
" print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
" continue\n",
" return {\"documents\": filtered_docs, \"question\": question}\n",
"\n",
"\n",
"def transform_query(state):\n",
" \"\"\"\n",
" Transform the query to produce a better question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates question key with a re-phrased question\n",
" \"\"\"\n",
"\n",
" print(\"---TRANSFORM QUERY---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Re-write question\n",
" better_question = question_rewriter.invoke({\"question\": question})\n",
" return {\"documents\": documents, \"question\": better_question}\n",
"\n",
"\n",
"### Edges\n",
"\n",
"\n",
"def decide_to_generate(state):\n",
" \"\"\"\n",
" Determines whether to generate an answer, or re-generate a question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Binary decision for next node to call\n",
" \"\"\"\n",
"\n",
" print(\"---ASSESS GRADED DOCUMENTS---\")\n",
" state[\"question\"]\n",
" filtered_documents = state[\"documents\"]\n",
"\n",
" if not filtered_documents:\n",
" # All documents have been filtered check_relevance\n",
" # We will re-generate a new query\n",
" print(\n",
" \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
" )\n",
" return \"transform_query\"\n",
" else:\n",
" # We have relevant documents, so generate answer\n",
" print(\"---DECISION: GENERATE---\")\n",
" return \"generate\"\n",
"\n",
"\n",
"def grade_generation_v_documents_and_question(state):\n",
" \"\"\"\n",
" Determines whether the generation is grounded in the document and answers question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Decision for next node to call\n",
" \"\"\"\n",
"\n",
" print(\"---CHECK HALLUCINATIONS---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
" generation = state[\"generation\"]\n",
"\n",
" score = hallucination_grader.invoke(\n",
" {\"documents\": documents, \"generation\": generation}\n",
" )\n",
" grade = score.binary_score\n",
"\n",
" # Check hallucination\n",
" if grade == \"yes\":\n",
" print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
" # Check question-answering\n",
" print(\"---GRADE GENERATION vs QUESTION---\")\n",
" score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
" grade = score.binary_score\n",
" if grade == \"yes\":\n",
" print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
" return \"useful\"\n",
" else:\n",
" print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
" return \"not useful\"\n",
" else:\n",
" pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
" return \"not supported\""
]
"source": ["### Nodes\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.get_relevant_documents(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n continue\n return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}\n\n\n### Edges\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n filtered_documents = state[\"documents\"]\n\n if not filtered_documents:\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\"\n\n\ndef grade_generation_v_documents_and_question(state):\n \"\"\"\n Determines whether the generation is grounded in the document and answers question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Decision for next node to call\n \"\"\"\n\n print(\"---CHECK HALLUCINATIONS---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = state[\"generation\"]\n\n score = hallucination_grader.invoke(\n {\"documents\": documents, \"generation\": generation}\n )\n grade = score.binary_score\n\n # Check hallucination\n if grade == \"yes\":\n print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n # Check question-answering\n print(\"---GRADE GENERATION vs QUESTION---\")\n score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n return \"useful\"\n else:\n print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n return \"not useful\"\n else:\n pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n return \"not supported\""]
},
{
"cell_type": "markdown",
@@ -620,42 +262,7 @@
"id": "0e09ca9f-e36d-4ef4-a0d5-79fdbada9fe0",
"metadata": {},
"outputs": [],
"source": [
"from langgraph.graph import END, StateGraph\n",
"\n",
"workflow = StateGraph(GraphState)\n",
"\n",
"# Define the nodes\n",
"workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
"workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
"workflow.add_node(\"generate\", generate) # generatae\n",
"workflow.add_node(\"transform_query\", transform_query) # transform_query\n",
"\n",
"# Build graph\n",
"workflow.set_entry_point(\"retrieve\")\n",
"workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
"workflow.add_conditional_edges(\n",
" \"grade_documents\",\n",
" decide_to_generate,\n",
" {\n",
" \"transform_query\": \"transform_query\",\n",
" \"generate\": \"generate\",\n",
" },\n",
")\n",
"workflow.add_edge(\"transform_query\", \"retrieve\")\n",
"workflow.add_conditional_edges(\n",
" \"generate\",\n",
" grade_generation_v_documents_and_question,\n",
" {\n",
" \"not supported\": \"generate\",\n",
" \"useful\": END,\n",
" \"not useful\": \"transform_query\",\n",
" },\n",
")\n",
"\n",
"# Compile\n",
"app = workflow.compile()"
]
"source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"transform_query\", transform_query) # transform_query\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"transform_query\": \"transform_query\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"transform_query\", \"retrieve\")\nworkflow.add_conditional_edges(\n \"generate\",\n grade_generation_v_documents_and_question,\n {\n \"not supported\": \"generate\",\n \"useful\": END,\n \"not useful\": \"transform_query\",\n },\n)\n\n# Compile\napp = workflow.compile()"]
},
{
"cell_type": "code",
@@ -694,22 +301,7 @@
]
}
],
"source": [
"from pprint import pprint\n",
"\n",
"# Run\n",
"inputs = {\"question\": \"Explain how the different types of agent memory work?\"}\n",
"for output in app.stream(inputs):\n",
" for key, value in output.items():\n",
" # Node\n",
" pprint(f\"Node '{key}':\")\n",
" # Optional: print full state at each node\n",
" # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
" pprint(\"\\n---\\n\")\n",
"\n",
"# Final generation\n",
"pprint(value[\"generation\"])"
]
"source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"Explain how the different types of agent memory work?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
},
{
"cell_type": "code",
@@ -749,19 +341,7 @@
]
}
],
"source": [
"inputs = {\"question\": \"Explain how chain of thought prompting works?\"}\n",
"for output in app.stream(inputs):\n",
" for key, value in output.items():\n",
" # Node\n",
" pprint(f\"Node '{key}':\")\n",
" # Optional: print full state at each node\n",
" # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
" pprint(\"\\n---\\n\")\n",
"\n",
"# Final generation\n",
"pprint(value[\"generation\"])"
]
"source": ["inputs = {\"question\": \"Explain how chain of thought prompting works?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
},
{
"cell_type": "markdown",
@@ -781,7 +361,7 @@
"id": "42369ab8-322d-434a-b5dd-2266e4cb2903",
"metadata": {},
"outputs": [],
"source": []
"source": [""]
}
],
"metadata": {
examples/rag/langgraph_self_rag_local.ipynb
+14 -390
View File
@@ -59,10 +59,7 @@
"id": "d7f9cc6d-a70c-433a-b0ad-ea47c5a0717e",
"metadata": {},
"outputs": [],
"source": [
"%capture --no-stderr\n",
"%pip install -U langchain-nomic langchain_community tiktoken langchainhub chromadb langchain langgraph nomic[local]"
]
"source": ["%capture --no-stderr\n%pip install -U langchain-nomic langchain_community tiktoken langchainhub chromadb langchain langgraph nomic[local]"]
},
{
"cell_type": "markdown",
@@ -94,10 +91,7 @@
"id": "bedffc73-6b10-42c8-8768-2085c8ed3398",
"metadata": {},
"outputs": [],
"source": [
"# Ollama model name\n",
"local_llm = \"mistral\""
]
"source": ["# Ollama model name\nlocal_llm = \"mistral\""]
},
{
"cell_type": "markdown",
@@ -115,13 +109,7 @@
"id": "2208f342-8163-4af3-8dc0-aa70f5e06143",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
"os.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""
]
"source": ["import os\n\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""]
},
{
"cell_type": "markdown",
@@ -139,34 +127,7 @@
"id": "c3bb9060-ad74-4470-9991-2ba167b6b8d8",
"metadata": {},
"outputs": [],
"source": [
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"from langchain_community.document_loaders import WebBaseLoader\n",
"from langchain_community.vectorstores import Chroma\n",
"from langchain_nomic.embeddings import NomicEmbeddings\n",
"\n",
"urls = [\n",
" \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
" \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
" \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
"]\n",
"\n",
"docs = [WebBaseLoader(url).load() for url in urls]\n",
"docs_list = [item for sublist in docs for item in sublist]\n",
"\n",
"text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
" chunk_size=250, chunk_overlap=0\n",
")\n",
"doc_splits = text_splitter.split_documents(docs_list)\n",
"\n",
"# Add to vectorDB\n",
"vectorstore = Chroma.from_documents(\n",
" documents=doc_splits,\n",
" collection_name=\"rag-chroma\",\n",
" embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n",
")\n",
"retriever = vectorstore.as_retriever()"
]
"source": ["from langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\nfrom langchain_nomic.embeddings import NomicEmbeddings\n\nurls = [\n \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n chunk_size=250, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorDB\nvectorstore = Chroma.from_documents(\n documents=doc_splits,\n collection_name=\"rag-chroma\",\n embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n)\nretriever = vectorstore.as_retriever()"]
},
{
"cell_type": "markdown",
@@ -190,33 +151,7 @@
]
}
],
"source": [
"### Retrieval Grader\n",
"\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain_community.chat_models import ChatOllama\n",
"from langchain_core.output_parsers import JsonOutputParser\n",
"\n",
"# LLM\n",
"llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
"\n",
"prompt = PromptTemplate(\n",
" template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
" Here is the retrieved document: \\n\\n {document} \\n\\n\n",
" Here is the user question: {question} \\n\n",
" If the document contains keywords related to the user question, grade it as relevant. \\n\n",
" It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
" Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n",
" Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\"\"\",\n",
" input_variables=[\"question\", \"document\"],\n",
")\n",
"\n",
"retrieval_grader = prompt | llm | JsonOutputParser()\n",
"question = \"agent memory\"\n",
"docs = retriever.get_relevant_documents(question)\n",
"doc_txt = docs[1].page_content\n",
"print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
]
"source": ["### Retrieval Grader\n\nfrom langchain.prompts import PromptTemplate\nfrom langchain_community.chat_models import ChatOllama\nfrom langchain_core.output_parsers import JsonOutputParser\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n Here is the retrieved document: \\n\\n {document} \\n\\n\n Here is the user question: {question} \\n\n If the document contains keywords related to the user question, grade it as relevant. \\n\n It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\"\"\",\n input_variables=[\"question\", \"document\"],\n)\n\nretrieval_grader = prompt | llm | JsonOutputParser()\nquestion = \"agent memory\"\ndocs = retriever.get_relevant_documents(question)\ndoc_txt = docs[1].page_content\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"]
},
{
"cell_type": "code",
@@ -232,31 +167,7 @@
]
}
],
"source": [
"### Generate\n",
"\n",
"from langchain import hub\n",
"from langchain_core.output_parsers import StrOutputParser\n",
"\n",
"# Prompt\n",
"prompt = hub.pull(\"rlm/rag-prompt\")\n",
"\n",
"# LLM\n",
"llm = ChatOllama(model=local_llm, temperature=0)\n",
"\n",
"\n",
"# Post-processing\n",
"def format_docs(docs):\n",
" return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
"\n",
"\n",
"# Chain\n",
"rag_chain = prompt | llm | StrOutputParser()\n",
"\n",
"# Run\n",
"generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
"print(generation)"
]
"source": ["### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOllama(model=local_llm, temperature=0)\n\n\n# Post-processing\ndef format_docs(docs):\n return \"\\n\\n\".join(doc.page_content for doc in docs)\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"]
},
{
"cell_type": "code",
@@ -275,28 +186,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Hallucination Grader\n",
"\n",
"# LLM\n",
"llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
"\n",
"# Prompt\n",
"prompt = PromptTemplate(\n",
" template=\"\"\"You are a grader assessing whether an answer is grounded in / supported by a set of facts. \\n \n",
" Here are the facts:\n",
" \\n ------- \\n\n",
" {documents} \n",
" \\n ------- \\n\n",
" Here is the answer: {generation}\n",
" Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \\n\n",
" Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
" input_variables=[\"generation\", \"documents\"],\n",
")\n",
"\n",
"hallucination_grader = prompt | llm | JsonOutputParser()\n",
"hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
]
"source": ["### Hallucination Grader\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing whether an answer is grounded in / supported by a set of facts. \\n \n Here are the facts:\n \\n ------- \\n\n {documents} \n \\n ------- \\n\n Here is the answer: {generation}\n Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \\n\n Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n input_variables=[\"generation\", \"documents\"],\n)\n\nhallucination_grader = prompt | llm | JsonOutputParser()\nhallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"]
},
{
"cell_type": "code",
@@ -315,28 +205,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Answer Grader\n",
"\n",
"# LLM\n",
"llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
"\n",
"# Prompt\n",
"prompt = PromptTemplate(\n",
" template=\"\"\"You are a grader assessing whether an answer is useful to resolve a question. \\n \n",
" Here is the answer:\n",
" \\n ------- \\n\n",
" {generation} \n",
" \\n ------- \\n\n",
" Here is the question: {question}\n",
" Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \\n\n",
" Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
" input_variables=[\"generation\", \"question\"],\n",
")\n",
"\n",
"answer_grader = prompt | llm | JsonOutputParser()\n",
"answer_grader.invoke({\"question\": question, \"generation\": generation})"
]
"source": ["### Answer Grader\n\n# LLM\nllm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n\n# Prompt\nprompt = PromptTemplate(\n template=\"\"\"You are a grader assessing whether an answer is useful to resolve a question. \\n \n Here is the answer:\n \\n ------- \\n\n {generation} \n \\n ------- \\n\n Here is the question: {question}\n Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \\n\n Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n input_variables=[\"generation\", \"question\"],\n)\n\nanswer_grader = prompt | llm | JsonOutputParser()\nanswer_grader.invoke({\"question\": question, \"generation\": generation})"]
},
{
"cell_type": "code",
@@ -355,23 +224,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Question Re-writer\n",
"\n",
"# LLM\n",
"llm = ChatOllama(model=local_llm, temperature=0)\n",
"\n",
"# Prompt\n",
"re_write_prompt = PromptTemplate(\n",
" template=\"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
" for vectorstore retrieval. Look at the initial and formulate an improved question. \\n\n",
" Here is the initial question: \\n\\n {question}. Improved question with no preamble: \\n \"\"\",\n",
" input_variables=[\"generation\", \"question\"],\n",
")\n",
"\n",
"question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
"question_rewriter.invoke({\"question\": question})"
]
"source": ["### Question Re-writer\n\n# LLM\nllm = ChatOllama(model=local_llm, temperature=0)\n\n# Prompt\nre_write_prompt = PromptTemplate(\n template=\"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n for vectorstore retrieval. Look at the initial and formulate an improved question. \\n\n Here is the initial question: \\n\\n {question}. Improved question with no preamble: \\n \"\"\",\n input_variables=[\"generation\", \"question\"],\n)\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nquestion_rewriter.invoke({\"question\": question})"]
},
{
"cell_type": "markdown",
@@ -391,26 +244,7 @@
"id": "90fb1dc6-c482-483a-8441-39965c401beb",
"metadata": {},
"outputs": [],
"source": [
"from typing import List\n",
"\n",
"from typing_extensions import TypedDict\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
"\n",
" Attributes:\n",
" question: question\n",
" generation: LLM generation\n",
" documents: list of documents\n",
" \"\"\"\n",
"\n",
" question: str\n",
" generation: str\n",
" documents: List[str]"
]
"source": ["from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n documents: List[str]"]
},
{
"cell_type": "code",
@@ -418,167 +252,7 @@
"id": "5324ea49-5745-47b5-a0a5-bf58c8babe46",
"metadata": {},
"outputs": [],
"source": [
"### Nodes\n",
"\n",
"\n",
"def retrieve(state):\n",
" \"\"\"\n",
" Retrieve documents\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, documents, that contains retrieved documents\n",
" \"\"\"\n",
" print(\"---RETRIEVE---\")\n",
" question = state[\"question\"]\n",
"\n",
" # Retrieval\n",
" documents = retriever.get_relevant_documents(question)\n",
" return {\"documents\": documents, \"question\": question}\n",
"\n",
"\n",
"def generate(state):\n",
" \"\"\"\n",
" Generate answer\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, generation, that contains LLM generation\n",
" \"\"\"\n",
" print(\"---GENERATE---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # RAG generation\n",
" generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
" return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
"\n",
"\n",
"def grade_documents(state):\n",
" \"\"\"\n",
" Determines whether the retrieved documents are relevant to the question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates documents key with only filtered relevant documents\n",
" \"\"\"\n",
"\n",
" print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Score each doc\n",
" filtered_docs = []\n",
" for d in documents:\n",
" score = retrieval_grader.invoke(\n",
" {\"question\": question, \"document\": d.page_content}\n",
" )\n",
" grade = score[\"score\"]\n",
" if grade == \"yes\":\n",
" print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
" filtered_docs.append(d)\n",
" else:\n",
" print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
" continue\n",
" return {\"documents\": filtered_docs, \"question\": question}\n",
"\n",
"\n",
"def transform_query(state):\n",
" \"\"\"\n",
" Transform the query to produce a better question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates question key with a re-phrased question\n",
" \"\"\"\n",
"\n",
" print(\"---TRANSFORM QUERY---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Re-write question\n",
" better_question = question_rewriter.invoke({\"question\": question})\n",
" return {\"documents\": documents, \"question\": better_question}\n",
"\n",
"\n",
"### Edges\n",
"\n",
"\n",
"def decide_to_generate(state):\n",
" \"\"\"\n",
" Determines whether to generate an answer, or re-generate a question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Binary decision for next node to call\n",
" \"\"\"\n",
"\n",
" print(\"---ASSESS GRADED DOCUMENTS---\")\n",
" state[\"question\"]\n",
" filtered_documents = state[\"documents\"]\n",
"\n",
" if not filtered_documents:\n",
" # All documents have been filtered check_relevance\n",
" # We will re-generate a new query\n",
" print(\n",
" \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
" )\n",
" return \"transform_query\"\n",
" else:\n",
" # We have relevant documents, so generate answer\n",
" print(\"---DECISION: GENERATE---\")\n",
" return \"generate\"\n",
"\n",
"\n",
"def grade_generation_v_documents_and_question(state):\n",
" \"\"\"\n",
" Determines whether the generation is grounded in the document and answers question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Decision for next node to call\n",
" \"\"\"\n",
"\n",
" print(\"---CHECK HALLUCINATIONS---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
" generation = state[\"generation\"]\n",
"\n",
" score = hallucination_grader.invoke(\n",
" {\"documents\": documents, \"generation\": generation}\n",
" )\n",
" grade = score[\"score\"]\n",
"\n",
" # Check hallucination\n",
" if grade == \"yes\":\n",
" print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
" # Check question-answering\n",
" print(\"---GRADE GENERATION vs QUESTION---\")\n",
" score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
" grade = score[\"score\"]\n",
" if grade == \"yes\":\n",
" print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
" return \"useful\"\n",
" else:\n",
" print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
" return \"not useful\"\n",
" else:\n",
" pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
" return \"not supported\""
]
"source": ["### Nodes\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.get_relevant_documents(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score[\"score\"]\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n continue\n return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}\n\n\n### Edges\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n filtered_documents = state[\"documents\"]\n\n if not filtered_documents:\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\"\n\n\ndef grade_generation_v_documents_and_question(state):\n \"\"\"\n Determines whether the generation is grounded in the document and answers question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Decision for next node to call\n \"\"\"\n\n print(\"---CHECK HALLUCINATIONS---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = state[\"generation\"]\n\n score = hallucination_grader.invoke(\n {\"documents\": documents, \"generation\": generation}\n )\n grade = score[\"score\"]\n\n # Check hallucination\n if grade == \"yes\":\n print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n # Check question-answering\n print(\"---GRADE GENERATION vs QUESTION---\")\n score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n grade = score[\"score\"]\n if grade == \"yes\":\n print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n return \"useful\"\n else:\n print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n return \"not useful\"\n else:\n pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n return \"not supported\""]
},
{
"cell_type": "markdown",
@@ -596,42 +270,7 @@
"id": "5605dee4-b2df-46ae-a640-cc2ed90c21a6",
"metadata": {},
"outputs": [],
"source": [
"from langgraph.graph import END, StateGraph\n",
"\n",
"workflow = StateGraph(GraphState)\n",
"\n",
"# Define the nodes\n",
"workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
"workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
"workflow.add_node(\"generate\", generate) # generatae\n",
"workflow.add_node(\"transform_query\", transform_query) # transform_query\n",
"\n",
"# Build graph\n",
"workflow.set_entry_point(\"retrieve\")\n",
"workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
"workflow.add_conditional_edges(\n",
" \"grade_documents\",\n",
" decide_to_generate,\n",
" {\n",
" \"transform_query\": \"transform_query\",\n",
" \"generate\": \"generate\",\n",
" },\n",
")\n",
"workflow.add_edge(\"transform_query\", \"retrieve\")\n",
"workflow.add_conditional_edges(\n",
" \"generate\",\n",
" grade_generation_v_documents_and_question,\n",
" {\n",
" \"not supported\": \"generate\",\n",
" \"useful\": END,\n",
" \"not useful\": \"transform_query\",\n",
" },\n",
")\n",
"\n",
"# Compile\n",
"app = workflow.compile()"
]
"source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"transform_query\", transform_query) # transform_query\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"transform_query\": \"transform_query\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"transform_query\", \"retrieve\")\nworkflow.add_conditional_edges(\n \"generate\",\n grade_generation_v_documents_and_question,\n {\n \"not supported\": \"generate\",\n \"useful\": END,\n \"not useful\": \"transform_query\",\n },\n)\n\n# Compile\napp = workflow.compile()"]
},
{
"cell_type": "markdown",
@@ -685,22 +324,7 @@
]
}
],
"source": [
"from pprint import pprint\n",
"\n",
"# Run\n",
"inputs = {\"question\": \"Explain how the different types of agent memory work?\"}\n",
"for output in app.stream(inputs):\n",
" for key, value in output.items():\n",
" # Node\n",
" pprint(f\"Node '{key}':\")\n",
" # Optional: print full state at each node\n",
" # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
" pprint(\"\\n---\\n\")\n",
"\n",
"# Final generation\n",
"pprint(value[\"generation\"])"
]
"source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"Explain how the different types of agent memory work?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n # Optional: print full state at each node\n # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
},
{
"cell_type": "markdown",
@@ -718,7 +342,7 @@
"id": "953143c2-2f2a-4361-a36b-87db7cf21d63",
"metadata": {},
"outputs": [],
"source": []
"source": [""]
}
],
"metadata": {
examples/rag/langgraph_self_rag_pinecone_movies.ipynb
+18 -394
View File
@@ -33,9 +33,7 @@
"id": "a384cc48-0425-4e8f-aafc-cfb8e56025c9",
"metadata": {},
"outputs": [],
"source": [
"%pip install -qU langchain-pinecone langchain-openai langchainhub langgraph"
]
"source": ["%pip install -qU langchain-pinecone langchain-openai langchainhub langgraph"]
},
{
"cell_type": "markdown",
@@ -53,13 +51,7 @@
"id": "ccc3dae5-1df6-48ca-af8a-50f0e6128876",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
"os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
"os.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""
]
"source": ["import os\n\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""]
},
{
"cell_type": "code",
@@ -67,11 +59,7 @@
"id": "88637820",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"LANGCHAIN_PROJECT\"] = \"pinecone-devconnect\""
]
"source": ["import os\n\nos.environ[\"LANGCHAIN_PROJECT\"] = \"pinecone-devconnect\""]
},
{
"cell_type": "markdown",
@@ -89,20 +77,7 @@
"id": "565a6d44-2c9f-4fff-b1ec-eea05df9350d",
"metadata": {},
"outputs": [],
"source": [
"from langchain_openai import OpenAIEmbeddings\n",
"from langchain_pinecone import PineconeVectorStore\n",
"\n",
"# use pinecone movies database\n",
"\n",
"# Add to vectorDB\n",
"vectorstore = PineconeVectorStore(\n",
" embedding=OpenAIEmbeddings(),\n",
" index_name=\"sample-movies\",\n",
" text_key=\"summary\",\n",
")\n",
"retriever = vectorstore.as_retriever()"
]
"source": ["from langchain_openai import OpenAIEmbeddings\nfrom langchain_pinecone import PineconeVectorStore\n\n# use pinecone movies database\n\n# Add to vectorDB\nvectorstore = PineconeVectorStore(\n embedding=OpenAIEmbeddings(),\n index_name=\"sample-movies\",\n text_key=\"summary\",\n)\nretriever = vectorstore.as_retriever()"]
},
{
"cell_type": "code",
@@ -129,13 +104,7 @@
]
}
],
"source": [
"docs = retriever.invoke(\"James Cameron\")\n",
"for doc in docs:\n",
" print(\"# \" + doc.metadata[\"title\"])\n",
" print(doc.page_content)\n",
" print()"
]
"source": ["docs = retriever.invoke(\"James Cameron\")\nfor doc in docs:\n print(\"# \" + doc.metadata[\"title\"])\n print(doc.page_content)\n print()"]
},
{
"cell_type": "markdown",
@@ -151,32 +120,7 @@
"id": "1fafad21-60cc-483e-92a3-6a7edb1838e3",
"metadata": {},
"outputs": [],
"source": [
"### Retrieval Grader\n",
"\n",
"from langchain import hub\n",
"from langchain_core.pydantic_v1 import BaseModel, Field\n",
"from langchain_openai import ChatOpenAI\n",
"\n",
"\n",
"# Data model\n",
"class GradeDocuments(BaseModel):\n",
" \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
"\n",
" binary_score: str = Field(\n",
" description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
" )\n",
"\n",
"\n",
"# https://smith.langchain.com/hub/efriis/self-rag-retrieval-grader\n",
"grade_prompt = hub.pull(\"efriis/self-rag-retrieval-grader\")\n",
"\n",
"# LLM with function call\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
"\n",
"retrieval_grader = grade_prompt | structured_llm_grader"
]
"source": ["### Retrieval Grader\n\nfrom langchain import hub\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_openai import ChatOpenAI\n\n\n# Data model\nclass GradeDocuments(BaseModel):\n \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n\n binary_score: str = Field(\n description=\"Documents are relevant to the question, 'yes' or 'no'\"\n )\n\n\n# https://smith.langchain.com/hub/efriis/self-rag-retrieval-grader\ngrade_prompt = hub.pull(\"efriis/self-rag-retrieval-grader\")\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeDocuments)\n\nretrieval_grader = grade_prompt | structured_llm_grader"]
},
{
"cell_type": "code",
@@ -193,14 +137,7 @@
]
}
],
"source": [
"# Test the retrieval grader\n",
"question = \"movies starring jason momoa\"\n",
"docs = retriever.invoke(question)\n",
"doc_txt = docs[0].page_content\n",
"print(doc_txt)\n",
"print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
]
"source": ["# Test the retrieval grader\nquestion = \"movies starring jason momoa\"\ndocs = retriever.invoke(question)\ndoc_txt = docs[0].page_content\nprint(doc_txt)\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"]
},
{
"cell_type": "markdown",
@@ -226,25 +163,7 @@
]
}
],
"source": [
"### Generate\n",
"\n",
"from langchain import hub\n",
"from langchain_core.output_parsers import StrOutputParser\n",
"\n",
"# Prompt\n",
"prompt = hub.pull(\"rlm/rag-prompt\")\n",
"\n",
"# LLM\n",
"llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
"\n",
"# Chain\n",
"rag_chain = prompt | llm | StrOutputParser()\n",
"\n",
"# Run\n",
"generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
"print(generation)"
]
"source": ["### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"]
},
{
"cell_type": "code",
@@ -270,30 +189,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Hallucination Grader\n",
"\n",
"\n",
"# Data model\n",
"class GradeHallucinations(BaseModel):\n",
" \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n",
"\n",
" binary_score: str = Field(\n",
" description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n",
" )\n",
"\n",
"\n",
"# LLM with function call\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"structured_llm_grader = llm.with_structured_output(GradeHallucinations)\n",
"\n",
"# https://smith.langchain.com/hub/efriis/self-rag-hallucination-grader\n",
"hallucination_prompt = hub.pull(\"efriis/self-rag-hallucination-grader\")\n",
"\n",
"hallucination_grader = hallucination_prompt | structured_llm_grader\n",
"print(generation)\n",
"hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
]
"source": ["### Hallucination Grader\n\n\n# Data model\nclass GradeHallucinations(BaseModel):\n \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n\n binary_score: str = Field(\n description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeHallucinations)\n\n# https://smith.langchain.com/hub/efriis/self-rag-hallucination-grader\nhallucination_prompt = hub.pull(\"efriis/self-rag-hallucination-grader\")\n\nhallucination_grader = hallucination_prompt | structured_llm_grader\nprint(generation)\nhallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"]
},
{
"cell_type": "code",
@@ -320,31 +216,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Answer Grader\n",
"\n",
"\n",
"# Data model\n",
"class GradeAnswer(BaseModel):\n",
" \"\"\"Binary score to assess answer addresses question.\"\"\"\n",
"\n",
" binary_score: str = Field(\n",
" description=\"Answer addresses the question, 'yes' or 'no'\"\n",
" )\n",
"\n",
"\n",
"# LLM with function call\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"structured_llm_grader = llm.with_structured_output(GradeAnswer)\n",
"\n",
"# Prompt\n",
"answer_prompt = hub.pull(\"efriis/self-rag-answer-grader\")\n",
"\n",
"answer_grader = answer_prompt | structured_llm_grader\n",
"print(question)\n",
"print(generation)\n",
"answer_grader.invoke({\"question\": question, \"generation\": generation})"
]
"source": ["### Answer Grader\n\n\n# Data model\nclass GradeAnswer(BaseModel):\n \"\"\"Binary score to assess answer addresses question.\"\"\"\n\n binary_score: str = Field(\n description=\"Answer addresses the question, 'yes' or 'no'\"\n )\n\n\n# LLM with function call\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeAnswer)\n\n# Prompt\nanswer_prompt = hub.pull(\"efriis/self-rag-answer-grader\")\n\nanswer_grader = answer_prompt | structured_llm_grader\nprint(question)\nprint(generation)\nanswer_grader.invoke({\"question\": question, \"generation\": generation})"]
},
{
"cell_type": "code",
@@ -370,19 +242,7 @@
"output_type": "execute_result"
}
],
"source": [
"### Question Re-writer\n",
"\n",
"# LLM\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
"\n",
"# Prompt\n",
"re_write_prompt = hub.pull(\"efriis/self-rag-question-rewriter\")\n",
"\n",
"question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
"print(question)\n",
"question_rewriter.invoke({\"question\": question})"
]
"source": ["### Question Re-writer\n\n# LLM\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n\n# Prompt\nre_write_prompt = hub.pull(\"efriis/self-rag-question-rewriter\")\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nprint(question)\nquestion_rewriter.invoke({\"question\": question})"]
},
{
"cell_type": "markdown",
@@ -402,26 +262,7 @@
"id": "f1617e9e-66a8-4c1a-a1fe-cc936284c085",
"metadata": {},
"outputs": [],
"source": [
"from typing import List\n",
"\n",
"from typing_extensions import TypedDict\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
"\n",
" Attributes:\n",
" question: question\n",
" generation: LLM generation\n",
" documents: list of documents\n",
" \"\"\"\n",
"\n",
" question: str\n",
" generation: str\n",
" documents: List[str]"
]
"source": ["from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n \"\"\"\n Represents the state of our graph.\n\n Attributes:\n question: question\n generation: LLM generation\n documents: list of documents\n \"\"\"\n\n question: str\n generation: str\n documents: List[str]"]
},
{
"cell_type": "code",
@@ -429,97 +270,7 @@
"id": "add509d8-6682-4127-8d95-13dd37d79702",
"metadata": {},
"outputs": [],
"source": [
"### Nodes\n",
"\n",
"\n",
"def retrieve(state):\n",
" \"\"\"\n",
" Retrieve documents\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, documents, that contains retrieved documents\n",
" \"\"\"\n",
" print(\"---RETRIEVE---\")\n",
" question = state[\"question\"]\n",
"\n",
" # Retrieval\n",
" documents = retriever.invoke(question)\n",
" return {\"documents\": documents, \"question\": question}\n",
"\n",
"\n",
"def generate(state):\n",
" \"\"\"\n",
" Generate answer\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): New key added to state, generation, that contains LLM generation\n",
" \"\"\"\n",
" print(\"---GENERATE---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # RAG generation\n",
" generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
" return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
"\n",
"\n",
"def grade_documents(state):\n",
" \"\"\"\n",
" Determines whether the retrieved documents are relevant to the question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates documents key with only filtered relevant documents\n",
" \"\"\"\n",
"\n",
" print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Score each doc\n",
" filtered_docs = []\n",
" for d in documents:\n",
" score = retrieval_grader.invoke(\n",
" {\"question\": question, \"document\": d.page_content}\n",
" )\n",
" grade = score.binary_score\n",
" if grade == \"yes\":\n",
" print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
" filtered_docs.append(d)\n",
" else:\n",
" print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
" continue\n",
" return {\"documents\": filtered_docs, \"question\": question}\n",
"\n",
"\n",
"def transform_query(state):\n",
" \"\"\"\n",
" Transform the query to produce a better question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" state (dict): Updates question key with a re-phrased question\n",
" \"\"\"\n",
"\n",
" print(\"---TRANSFORM QUERY---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
"\n",
" # Re-write question\n",
" better_question = question_rewriter.invoke({\"question\": question})\n",
" return {\"documents\": documents, \"question\": better_question}"
]
"source": ["### Nodes\n\n\ndef retrieve(state):\n \"\"\"\n Retrieve documents\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, documents, that contains retrieved documents\n \"\"\"\n print(\"---RETRIEVE---\")\n question = state[\"question\"]\n\n # Retrieval\n documents = retriever.invoke(question)\n return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n \"\"\"\n Generate answer\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): New key added to state, generation, that contains LLM generation\n \"\"\"\n print(\"---GENERATE---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # RAG generation\n generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n \"\"\"\n Determines whether the retrieved documents are relevant to the question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates documents key with only filtered relevant documents\n \"\"\"\n\n print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Score each doc\n filtered_docs = []\n for d in documents:\n score = retrieval_grader.invoke(\n {\"question\": question, \"document\": d.page_content}\n )\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---GRADE: DOCUMENT RELEVANT---\")\n filtered_docs.append(d)\n else:\n print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n continue\n return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef transform_query(state):\n \"\"\"\n Transform the query to produce a better question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n state (dict): Updates question key with a re-phrased question\n \"\"\"\n\n print(\"---TRANSFORM QUERY---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n\n # Re-write question\n better_question = question_rewriter.invoke({\"question\": question})\n return {\"documents\": documents, \"question\": better_question}"]
},
{
"cell_type": "code",
@@ -527,76 +278,7 @@
"id": "09fc91b4",
"metadata": {},
"outputs": [],
"source": [
"### Edges\n",
"\n",
"\n",
"def decide_to_generate(state):\n",
" \"\"\"\n",
" Determines whether to generate an answer, or re-generate a question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Binary decision for next node to call\n",
" \"\"\"\n",
"\n",
" print(\"---ASSESS GRADED DOCUMENTS---\")\n",
" state[\"question\"]\n",
" filtered_documents = state[\"documents\"]\n",
"\n",
" if not filtered_documents:\n",
" # All documents have been filtered check_relevance\n",
" # We will re-generate a new query\n",
" print(\n",
" \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
" )\n",
" return \"transform_query\"\n",
" else:\n",
" # We have relevant documents, so generate answer\n",
" print(\"---DECISION: GENERATE---\")\n",
" return \"generate\"\n",
"\n",
"\n",
"def grade_generation_v_documents_and_question(state):\n",
" \"\"\"\n",
" Determines whether the generation is grounded in the document and answers question.\n",
"\n",
" Args:\n",
" state (dict): The current graph state\n",
"\n",
" Returns:\n",
" str: Decision for next node to call\n",
" \"\"\"\n",
"\n",
" print(\"---CHECK HALLUCINATIONS---\")\n",
" question = state[\"question\"]\n",
" documents = state[\"documents\"]\n",
" generation = state[\"generation\"]\n",
"\n",
" score = hallucination_grader.invoke(\n",
" {\"documents\": documents, \"generation\": generation}\n",
" )\n",
" grade = score.binary_score\n",
"\n",
" # Check hallucination\n",
" if grade == \"yes\":\n",
" print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
" # Check question-answering\n",
" print(\"---GRADE GENERATION vs QUESTION---\")\n",
" score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
" grade = score.binary_score\n",
" if grade == \"yes\":\n",
" print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
" return \"useful\"\n",
" else:\n",
" print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
" return \"not useful\"\n",
" else:\n",
" pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
" return \"not supported\""
]
"source": ["### Edges\n\n\ndef decide_to_generate(state):\n \"\"\"\n Determines whether to generate an answer, or re-generate a question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Binary decision for next node to call\n \"\"\"\n\n print(\"---ASSESS GRADED DOCUMENTS---\")\n state[\"question\"]\n filtered_documents = state[\"documents\"]\n\n if not filtered_documents:\n # All documents have been filtered check_relevance\n # We will re-generate a new query\n print(\n \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n )\n return \"transform_query\"\n else:\n # We have relevant documents, so generate answer\n print(\"---DECISION: GENERATE---\")\n return \"generate\"\n\n\ndef grade_generation_v_documents_and_question(state):\n \"\"\"\n Determines whether the generation is grounded in the document and answers question.\n\n Args:\n state (dict): The current graph state\n\n Returns:\n str: Decision for next node to call\n \"\"\"\n\n print(\"---CHECK HALLUCINATIONS---\")\n question = state[\"question\"]\n documents = state[\"documents\"]\n generation = state[\"generation\"]\n\n score = hallucination_grader.invoke(\n {\"documents\": documents, \"generation\": generation}\n )\n grade = score.binary_score\n\n # Check hallucination\n if grade == \"yes\":\n print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n # Check question-answering\n print(\"---GRADE GENERATION vs QUESTION---\")\n score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n grade = score.binary_score\n if grade == \"yes\":\n print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n return \"useful\"\n else:\n print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n return \"not useful\"\n else:\n pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n return \"not supported\""]
},
{
"cell_type": "markdown",
@@ -614,42 +296,7 @@
"id": "0e09ca9f-e36d-4ef4-a0d5-79fdbada9fe0",
"metadata": {},
"outputs": [],
"source": [
"from langgraph.graph import END, StateGraph\n",
"\n",
"workflow = StateGraph(GraphState)\n",
"\n",
"# Define the nodes\n",
"workflow.add_node(\"retrieve\", retrieve) # retrieve\n",
"workflow.add_node(\"grade_documents\", grade_documents) # grade documents\n",
"workflow.add_node(\"generate\", generate) # generatae\n",
"workflow.add_node(\"transform_query\", transform_query) # transform_query\n",
"\n",
"# Build graph\n",
"workflow.set_entry_point(\"retrieve\")\n",
"workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
"workflow.add_conditional_edges(\n",
" \"grade_documents\",\n",
" decide_to_generate,\n",
" {\n",
" \"transform_query\": \"transform_query\",\n",
" \"generate\": \"generate\",\n",
" },\n",
")\n",
"workflow.add_edge(\"transform_query\", \"retrieve\")\n",
"workflow.add_conditional_edges(\n",
" \"generate\",\n",
" grade_generation_v_documents_and_question,\n",
" {\n",
" \"not supported\": \"generate\",\n",
" \"useful\": END,\n",
" \"not useful\": \"transform_query\",\n",
" },\n",
")\n",
"\n",
"# Compile\n",
"app = workflow.compile()"
]
"source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve) # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents) # grade documents\nworkflow.add_node(\"generate\", generate) # generatae\nworkflow.add_node(\"transform_query\", transform_query) # transform_query\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n \"grade_documents\",\n decide_to_generate,\n {\n \"transform_query\": \"transform_query\",\n \"generate\": \"generate\",\n },\n)\nworkflow.add_edge(\"transform_query\", \"retrieve\")\nworkflow.add_conditional_edges(\n \"generate\",\n grade_generation_v_documents_and_question,\n {\n \"not supported\": \"generate\",\n \"useful\": END,\n \"not useful\": \"transform_query\",\n },\n)\n\n# Compile\napp = workflow.compile()"]
},
{
"cell_type": "code",
@@ -684,20 +331,7 @@
]
}
],
"source": [
"from pprint import pprint\n",
"\n",
"# Run\n",
"inputs = {\"question\": \"Movies that star Daniel Craig\"}\n",
"for output in app.stream(inputs):\n",
" for key, value in output.items():\n",
" # Node\n",
" pprint(f\"Node '{key}':\")\n",
" pprint(\"\\n---\\n\")\n",
"\n",
"# Final generation\n",
"pprint(value[\"generation\"])"
]
"source": ["from pprint import pprint\n\n# Run\ninputs = {\"question\": \"Movies that star Daniel Craig\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
},
{
"cell_type": "code",
@@ -705,17 +339,7 @@
"id": "4138bc51-8c84-4b8a-8d24-f7f470721f6f",
"metadata": {},
"outputs": [],
"source": [
"inputs = {\"question\": \"Which movies are about aliens?\"}\n",
"for output in app.stream(inputs):\n",
" for key, value in output.items():\n",
" # Node\n",
" pprint(f\"Node '{key}':\")\n",
" pprint(\"\\n---\\n\")\n",
"\n",
"# Final generation\n",
"pprint(value[\"generation\"])"
]
"source": ["inputs = {\"question\": \"Which movies are about aliens?\"}\nfor output in app.stream(inputs):\n for key, value in output.items():\n # Node\n pprint(f\"Node '{key}':\")\n pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"]
},
{
"cell_type": "code",
@@ -723,7 +347,7 @@
"id": "42369ab8-322d-434a-b5dd-2266e4cb2903",
"metadata": {},
"outputs": [],
"source": []
"source": [""]
}
],
"metadata": {