add test

.github/workflows/_test_langgraph.yml

@@ -56,3 +56,46 @@ jobs:
           # grep will exit non-zero if the target message isn't found,
           # and `set -e` above will cause the step to fail.
           echo "$STATUS" | grep 'nothing to commit, working tree clean'
+
+  build-strict-msgpack:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: libs/langgraph
+    name: "test strict msgpack #3.12"
+    steps:
+      - uses: actions/checkout@v6
+      - name: Set up Python 3.12
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.12"
+          enable-cache: true
+          cache-suffix: "test-langgraph-strict-msgpack"
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: ${{ !github.event.pull_request.head.repo.fork }}
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_RO_TOKEN }}
+
+      - name: Install dependencies
+        shell: bash
+        run: uv sync --frozen --group test --no-dev
+
+      - name: Run tests (strict msgpack)
+        shell: bash
+        env:
+          LANGGRAPH_STRICT_MSGPACK: "1"
+        run: make test_parallel
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

examples/README.md

@@ -0,0 +1,3 @@
+# LangGraph examples
+
+This directory should NOT be used for documentation. All new documentation must be added to `docs/docs/` directory.

examples/async.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "23544406",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/async.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/branching.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "14f7ca50",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/branching.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/chatbot-simulation-evaluation/agent-simulation-evaluation.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "10251c1c",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/chatbot-simulation-evaluation/agent-simulation-evaluation.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/chatbot-simulation-evaluation/langsmith-agent-simulation-evaluation.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a4351a24",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/chatbot-simulation-evaluation/langsmith-agent-simulation-evaluation.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/chatbot-simulation-evaluation/simulation_utils.py

@@ -0,0 +1,203 @@
+import functools
+from typing import Annotated, Any, Callable, Dict, List, Optional, Union
+
+from langchain_community.adapters.openai import convert_message_to_dict
+from langchain_core.messages import AIMessage, AnyMessage, BaseMessage, HumanMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_core.runnables import Runnable, RunnableLambda
+from langchain_core.runnables import chain as as_runnable
+from langchain_openai import ChatOpenAI
+from typing_extensions import TypedDict
+
+from langgraph.graph import END, StateGraph, START
+
+
+def langchain_to_openai_messages(messages: List[BaseMessage]):
+    """
+    Convert a list of langchain base messages to a list of openai messages.
+
+    Parameters:
+        messages (List[BaseMessage]): A list of langchain base messages.
+
+    Returns:
+        List[dict]: A list of openai messages.
+    """
+
+    return [
+        convert_message_to_dict(m) if isinstance(m, BaseMessage) else m
+        for m in messages
+    ]
+
+
+def create_simulated_user(
+    system_prompt: str, llm: Runnable | None = None
+) -> Runnable[Dict, AIMessage]:
+    """
+    Creates a simulated user for chatbot simulation.
+
+    Args:
+        system_prompt (str): The system prompt to be used by the simulated user.
+        llm (Runnable | None, optional): The language model to be used for the simulation.
+            Defaults to gpt-3.5-turbo.
+
+    Returns:
+        Runnable[Dict, AIMessage]: The simulated user for chatbot simulation.
+    """
+    return ChatPromptTemplate.from_messages(
+        [
+            ("system", system_prompt),
+            MessagesPlaceholder(variable_name="messages"),
+        ]
+    ) | (llm or ChatOpenAI(model="gpt-3.5-turbo")).with_config(
+        run_name="simulated_user"
+    )
+
+
+Messages = Union[list[AnyMessage], AnyMessage]
+
+
+def add_messages(left: Messages, right: Messages) -> Messages:
+    if not isinstance(left, list):
+        left = [left]
+    if not isinstance(right, list):
+        right = [right]
+    return left + right
+
+
+class SimulationState(TypedDict):
+    """
+    Represents the state of a simulation.
+
+    Attributes:
+        messages (List[AnyMessage]): A list of messages in the simulation.
+        inputs (Optional[dict[str, Any]]): Optional inputs for the simulation.
+    """
+
+    messages: Annotated[List[AnyMessage], add_messages]
+    inputs: Optional[dict[str, Any]]
+
+
+def create_chat_simulator(
+    assistant: (
+        Callable[[List[AnyMessage]], str | AIMessage]
+        | Runnable[List[AnyMessage], str | AIMessage]
+    ),
+    simulated_user: Runnable[Dict, AIMessage],
+    *,
+    input_key: str,
+    max_turns: int = 6,
+    should_continue: Optional[Callable[[SimulationState], str]] = None,
+):
+    """Creates a chat simulator for evaluating a chatbot.
+
+    Args:
+        assistant: The chatbot assistant function or runnable object.
+        simulated_user: The simulated user object.
+        input_key: The key for the input to the chat simulation.
+        max_turns: The maximum number of turns in the chat simulation. Default is 6.
+        should_continue: Optional function to determine if the simulation should continue.
+            If not provided, a default function will be used.
+
+    Returns:
+        The compiled chat simulation graph.
+
+    """
+    graph_builder = StateGraph(SimulationState)
+    graph_builder.add_node(
+        "user",
+        _create_simulated_user_node(simulated_user),
+    )
+    graph_builder.add_node(
+        "assistant", _fetch_messages | assistant | _coerce_to_message
+    )
+    graph_builder.add_edge("assistant", "user")
+    graph_builder.add_conditional_edges(
+        "user",
+        should_continue or functools.partial(_should_continue, max_turns=max_turns),
+    )
+    # If your dataset has a 'leading question/input', then we route first to the assistant, otherwise, we let the user take the lead.
+    graph_builder.add_edge(START, "assistant" if input_key is not None else "user")
+
+    return (
+        RunnableLambda(_prepare_example).bind(input_key=input_key)
+        | graph_builder.compile()
+    )
+
+
+## Private methods
+
+
+def _prepare_example(inputs: dict[str, Any], input_key: Optional[str] = None):
+    if input_key is not None:
+        if input_key not in inputs:
+            raise ValueError(
+                f"Dataset's example input must contain the provided input key: '{input_key}'.\nFound: {list(inputs.keys())}"
+            )
+        messages = [HumanMessage(content=inputs[input_key])]
+        return {
+            "inputs": {k: v for k, v in inputs.items() if k != input_key},
+            "messages": messages,
+        }
+    return {"inputs": inputs, "messages": []}
+
+
+def _invoke_simulated_user(state: SimulationState, simulated_user: Runnable):
+    """Invoke the simulated user node."""
+    runnable = (
+        simulated_user
+        if isinstance(simulated_user, Runnable)
+        else RunnableLambda(simulated_user)
+    )
+    inputs = state.get("inputs", {})
+    inputs["messages"] = state["messages"]
+    return runnable.invoke(inputs)
+
+
+def _swap_roles(state: SimulationState):
+    new_messages = []
+    for m in state["messages"]:
+        if isinstance(m, AIMessage):
+            new_messages.append(HumanMessage(content=m.content))
+        else:
+            new_messages.append(AIMessage(content=m.content))
+    return {
+        "inputs": state.get("inputs", {}),
+        "messages": new_messages,
+    }
+
+
+@as_runnable
+def _fetch_messages(state: SimulationState):
+    """Invoke the simulated user node."""
+    return state["messages"]
+
+
+def _convert_to_human_message(message: BaseMessage):
+    return {"messages": [HumanMessage(content=message.content)]}
+
+
+def _create_simulated_user_node(simulated_user: Runnable):
+    """Simulated user accepts a {"messages": [...]} argument and returns a single message."""
+    return (
+        _swap_roles
+        | RunnableLambda(_invoke_simulated_user).bind(simulated_user=simulated_user)
+        | _convert_to_human_message
+    )
+
+
+def _coerce_to_message(assistant_output: str | BaseMessage):
+    if isinstance(assistant_output, str):
+        return {"messages": [AIMessage(content=assistant_output)]}
+    else:
+        return {"messages": [assistant_output]}
+
+
+def _should_continue(state: SimulationState, max_turns: int = 6):
+    messages = state["messages"]
+    # TODO support other stop criteria
+    if len(messages) > max_turns:
+        return END
+    elif messages[-1].content.strip() == "FINISHED":
+        return END
+    else:
+        return "assistant"

examples/chatbots/information-gather-prompting.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a9014f94",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/chatbots/information-gather-prompting.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/cloud_examples/langgraph_to_langgraph_cloud.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2b789e16",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/cloud/how-tos/langgraph_to_langgraph_cloud.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/code_assistant/langgraph_code_assistant.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "1f2f13ca",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/code_assistant/langgraph_code_assistant.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/code_assistant/langgraph_code_assistant_mistral.ipynb

@@ -0,0 +1,298 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "15d3ac32-cdf3-4800-a30c-f26d828d69c8.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABq8AAANXCAYAAACrDsXfAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAavoAMABAAAAAEAAANXAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdN7np1MAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjg1NTwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNzExPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "6101c9d6-b7d1-46af-afab-47d05bfadeae",
+   "metadata": {},
+   "source": [
+    "# Code generation with self-correction\n",
+    "\n",
+    "AlphaCodium presented an approach for code generation that uses control flow.\n",
+    "\n",
+    "Main idea: [construct an answer to a coding question iteratively.](https://x.com/karpathy/status/1748043513156272416?s=20). \n",
+    "\n",
+    "[AlphaCodium](https://github.com/Codium-ai/AlphaCodium) iteravely tests and improves an answer on public and AI-generated tests for a particular question. \n",
+    "\n",
+    "We will implement some of these ideas from scratch using [LangGraph](https://langchain-ai.github.io/langgraph/):\n",
+    "\n",
+    "1. We show how to route user questions to different types of documentation\n",
+    "2. We we will show how to perform inline unit tests to confirm imports and code execution work\n",
+    "3. We will show how to use LangGraph to orchestrate this\n",
+    "\n",
+    "![Screenshot 2024-05-23 at 2.17.51 PM.png](attachment:15d3ac32-cdf3-4800-a30c-f26d828d69c8.png)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e501686f-323f-4b87-8f9c-8ba89133078b",
+   "metadata": {},
+   "outputs": [],
+   "source": ["! pip install -U langchain_community langchain-mistralai langchain langgraph"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9ef4fb67-113a-4b88-9f93-7e3a95cee035",
+   "metadata": {},
+   "source": [
+    "### LLM\n",
+    "\n",
+    "We'll use the Mistral API and `Codestral` instruct model, which support tool use!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "982e4609-86e4-4934-828f-e03d89c20393",
+   "metadata": {},
+   "outputs": [],
+   "source": ["import os\n\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"true\"\nmistral_api_key = os.getenv(\"MISTRAL_API_KEY\")  # Ensure this is set"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6a20a3eb-6dc5-4a61-9705-9daf68172c0b",
+   "metadata": {},
+   "source": [
+    "### Tracing\n",
+    "\n",
+    "Optionally, we'll use LangSmith for tracing."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "37b172d2-3a9d-49a8-898c-22ed0cb45c88",
+   "metadata": {},
+   "outputs": [],
+   "source": ["os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\"\nos.environ[\"LANGCHAIN_PROJECT\"] = \"Mistral-code-gen-testing\""]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d3a3dca9-4485-4ae5-aa87-cd1f02bad8b9",
+   "metadata": {},
+   "source": [
+    "## Code Generation\n",
+    "\n",
+    "Test with structured output."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "a188c8ca-c053-4e6d-b7af-38a3b6b371c7",
+   "metadata": {},
+   "outputs": [],
+   "source": ["# Select LLM\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\nfrom langchain_mistralai import ChatMistralAI\n\nmistral_model = \"mistral-large-latest\"\nllm = ChatMistralAI(model=mistral_model, temperature=0)\n\n# Prompt\ncode_gen_prompt_claude = ChatPromptTemplate.from_messages(\n    [\n        (\n            \"system\",\n            \"\"\"You are a coding assistant. Ensure any code you provide can be executed with all required imports and variables \\n\n            defined. Structure your answer: 1) a prefix describing the code solution, 2) the imports, 3) the functioning code block.\n            \\n Here is the user question:\"\"\",\n        ),\n        (\"placeholder\", \"{messages}\"),\n    ]\n)\n\n\n# Data model\nclass code(BaseModel):\n    \"\"\"Code output\"\"\"\n\n    prefix: str = Field(description=\"Description of the problem and approach\")\n    imports: str = Field(description=\"Code block import statements\")\n    code: str = Field(description=\"Code block not including import statements\")\n    description = \"Schema for code solutions to questions about LCEL.\"\n\n\n# LLM\ncode_gen_chain = llm.with_structured_output(code, include_raw=False)"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "9fc0290d-5a04-4514-8664-91f9dbf2da7b",
+   "metadata": {},
+   "outputs": [],
+   "source": ["question = \"Write a function for fibonacci.\"\nmessages = [(\"user\", question)]"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "973281bd-e74b-4386-98c6-210af5e31982",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "code(prefix='A function to calculate the nth Fibonacci number.', imports='', code='def fibonacci(n):\\n    if n <= 0:\\n        return \"Input should be positive integer\"\\n    elif n == 1:\\n        return 0\\n    elif n == 2:\\n        return 1\\n    else:\\n        a, b = 0, 1\\n        for _ in range(2, n):\\n            a, b = b, a + b\\n        return b', description='Schema for code solutions to questions about LCEL.')"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": ["# Test\nresult = code_gen_chain.invoke(messages)\nresult"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4235eb2d-f5b3-4cd0-bbb1-a889eb5564d7",
+   "metadata": {},
+   "source": [
+    "## State"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "183d77b8-f180-4815-b39f-8ef507ec0534",
+   "metadata": {},
+   "outputs": [],
+   "source": ["from typing import Annotated, TypedDict\n\nfrom langgraph.graph.message import AnyMessage, add_messages\n\n\nclass GraphState(TypedDict):\n    \"\"\"\n    Represents the state of our graph.\n\n    Attributes:\n        error : Binary flag for control flow to indicate whether test error was tripped\n        messages : With user question, error messages, reasoning\n        generation : Code solution\n        iterations : Number of tries\n    \"\"\"\n\n    error: str\n    messages: Annotated[list[AnyMessage], add_messages]\n    generation: str\n    iterations: int"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "55043d78-c012-4280-bc8b-259f04a29cb4",
+   "metadata": {},
+   "source": [
+    "## Graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "14bc89d1-3ca6-4847-a048-1803e0e4600e",
+   "metadata": {},
+   "outputs": [],
+   "source": ["import uuid\n\nfrom langchain_core.pydantic_v1 import BaseModel, Field\n\n### Parameters\nmax_iterations = 3\n\n\n### Nodes\ndef generate(state: GraphState):\n    \"\"\"\n    Generate a code solution\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): New key added to state, generation\n    \"\"\"\n\n    print(\"---GENERATING CODE SOLUTION---\")\n\n    # State\n    messages = state[\"messages\"]\n    iterations = state[\"iterations\"]\n\n    # Solution\n    code_solution = code_gen_chain.invoke(messages)\n    messages += [\n        (\n            \"assistant\",\n            f\"Here is my attempt to solve the problem: {code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n        )\n    ]\n\n    # Increment\n    iterations = iterations + 1\n    return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n\n\ndef code_check(state: GraphState):\n    \"\"\"\n    Check code\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): New key added to state, error\n    \"\"\"\n\n    print(\"---CHECKING CODE---\")\n\n    # State\n    messages = state[\"messages\"]\n    code_solution = state[\"generation\"]\n    iterations = state[\"iterations\"]\n\n    # Get solution components\n    imports = code_solution.imports\n    code = code_solution.code\n\n    # Check imports\n    try:\n        exec(imports)\n    except Exception as e:\n        print(\"---CODE IMPORT CHECK: FAILED---\")\n        error_message = [\n            (\n                \"user\",\n                f\"Your solution failed the import test. Here is the error: {e}. Reflect on this error and your prior attempt to solve the problem. (1) State what you think went wrong with the prior solution and (2) try to solve this problem again. Return the FULL SOLUTION. Use the code tool to structure the output with a prefix, imports, and code block:\",\n            )\n        ]\n        messages += error_message\n        return {\n            \"generation\": code_solution,\n            \"messages\": messages,\n            \"iterations\": iterations,\n            \"error\": \"yes\",\n        }\n\n    # Check execution\n    try:\n        combined_code = f\"{imports}\\n{code}\"\n        print(f\"CODE TO TEST: {combined_code}\")\n        # Use a shared scope for exec\n        global_scope = {}\n        exec(combined_code, global_scope)\n    except Exception as e:\n        print(\"---CODE BLOCK CHECK: FAILED---\")\n        error_message = [\n            (\n                \"user\",\n                f\"Your solution failed the code execution test: {e}) Reflect on this error and your prior attempt to solve the problem. (1) State what you think went wrong with the prior solution and (2) try to solve this problem again. Return the FULL SOLUTION. Use the code tool to structure the output with a prefix, imports, and code block:\",\n            )\n        ]\n        messages += error_message\n        return {\n            \"generation\": code_solution,\n            \"messages\": messages,\n            \"iterations\": iterations,\n            \"error\": \"yes\",\n        }\n\n    # No errors\n    print(\"---NO CODE TEST FAILURES---\")\n    return {\n        \"generation\": code_solution,\n        \"messages\": messages,\n        \"iterations\": iterations,\n        \"error\": \"no\",\n    }\n\n\n### Conditional edges\n\n\ndef decide_to_finish(state: GraphState):\n    \"\"\"\n    Determines whether to finish.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        str: Next node to call\n    \"\"\"\n    error = state[\"error\"]\n    iterations = state[\"iterations\"]\n\n    if error == \"no\" or iterations == max_iterations:\n        print(\"---DECISION: FINISH---\")\n        return \"end\"\n    else:\n        print(\"---DECISION: RE-TRY SOLUTION---\")\n        return \"generate\"\n\n\n### Utilities\n\n\ndef _print_event(event: dict, _printed: set, max_length=1500):\n    current_state = event.get(\"dialog_state\")\n    if current_state:\n        print(\"Currently in: \", current_state[-1])\n    message = event.get(\"messages\")\n    if message:\n        if isinstance(message, list):\n            message = message[-1]\n        if message.id not in _printed:\n            msg_repr = message.pretty_repr(html=True)\n            if len(msg_repr) > max_length:\n                msg_repr = msg_repr[:max_length] + \" ... (truncated)\"\n            print(msg_repr)\n            _printed.add(message.id)"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "2dff2209-44c7-4e2c-b607-ba6675f9e45f",
+   "metadata": {},
+   "outputs": [],
+   "source": ["from langgraph.checkpoint.memory import InMemorySaver\nfrom langgraph.graph import END, StateGraph, START\n\nbuilder = StateGraph(GraphState)\n\n# Define the nodes\nbuilder.add_node(\"generate\", generate)  # generation solution\nbuilder.add_node(\"check_code\", code_check)  # check code\n\n# Build graph\nbuilder.add_edge(START, \"generate\")\nbuilder.add_edge(\"generate\", \"check_code\")\nbuilder.add_conditional_edges(\n    \"check_code\",\n    decide_to_finish,\n    {\n        \"end\": END,\n        \"generate\": \"generate\",\n    },\n)\n\nmemory = InMemorySaver()\ngraph = builder.compile(checkpointer=memory)"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "d4bb21cd-af20-4d4d-89ff-384db034b7c3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAFBAHsDASIAAhEBAxEB/8QAHQABAAMAAwEBAQAAAAAAAAAAAAUGBwMECAkCAf/EAE8QAAEDAwEDBQwFBwoFBQAAAAECAwQABQYRBxIhCBMWMVUUFRciMkFRYZOU0eE2cXOz0iM1UlSBkZIJM0JWYnR1drGyJCZFU6FXhZWiwf/EABsBAQACAwEBAAAAAAAAAAAAAAACBAEDBQYH/8QAPBEAAgECAQgGBwgBBQAAAAAAAAECAxEhBBITFBUxUVJBU2GRobEFcZLB0uHwIjIzNEJigdFyQ2OCsvH/2gAMAwEAAhEDEQA/APqnSlKA6s66wrZud2TGIm/ru8+6lG9p16anj1j99dXpVZe2IHvKPjVK2kQ487OMZbksNyEC33BQS6gKAPOQ+OhrodHrX2bD9gj4VUynLKOSuMZxbbV8LcWvcdKhkemgp51jROlVl7Yge8o+NOlVl7Yge8o+NZ30etfZsP2CPhTo9a+zYfsEfCqm1cn5Jd6N+zv3eBonSqy9sQPeUfGnSqy9sQPeUfGs76PWvs2H7BHwp0etfZsP2CPhTauT8ku9DZ37vA0TpVZe2IHvKPjTpVZe2IHvKPjWd9HrX2bD9gj4U6PWvs2H7BHwptXJ+SXehs793gaJ0qsvbED3lHxp0qsvbED3lHxrO+j1r7Nh+wR8KdHrX2bD9gj4U2rk/JLvQ2d+7wNFRk9ncWlCLtBUtR0CUyUEk+jrqTrEcpstvjWguNQIzTiZEcpWhlII/LI6iBW3V0qNaGUUlVgmsWsexJ+8o5RQ0DSve4pSlbCoKUpQGd5/9PMa/wANuH3sOuOuTP8A6eY1/htw+9h1x15v0t+LD/H3yPSZF+Av5FQmYZpZcCsi7vfpyYEBK0Nc4UKWpa1HdShCEAqWok6BKQSam6oG262Wq64Rzd2td9uTLUth9leNtKcnxHkK3m5DQT42qCNeAP1HqrjQSlJJ7i5JtRbRBZhykMexuPhkyI1NuduyK4OQ+fat8srYQ2hZcVzQZKysLQE82QFcVEAhCqsOV7ccLwd6I1fLs7AXJjImJCoElQbZUSErdKWyGhqCPym7podeqsjfk5zOwzAMkyGzXi7uY/lrkhaUW7dub9t5qQy1Icio4hz8ogqQka6cdOuv3tYfyLNL1eYsq2ZsqxXHH2u8FusjLsZt2U6hwPJnrSU82pJLQ3HVBG7vcCdauaGDaXrvj2+oraWdm/V0dhr2S7Z8PxK7x7Xcbss3KRDTPYiw4b8tx5hSikLQGUK3xqlXVqQBqeHGonBtuVszXaHlWJohzY0mzzO5GXVQZPNvgNJWtSlloIb0UopCVK8YAKTqFCqVsXx+6M7QMQuM6zXCGiNs3g211+ZEW1zUlD/5RklQGi/F13esjQ9RBqwYO/OxLbbn1unWO7mPkU+PPgXViEtyEW0wm21hx4eK2oLaUNFaE6p066g6cI5yWLtx7SSnN2fRc2GlKVTLRC5f+Y1/bx/vkVsVY7l/5jX9vH++RWxV6/0Z+T/5S8onC9Iffj6hSlK6JyRSlKAzvP8A6eY1/htw+9h1W8r2f4znQi9I7Bbb73Lvcx3wiof5re03t3eB013U66egVo+T4TCyqXClSJEyLIhodbbchvc2d1woKgeHHi2j91RXgqg9sXv335VRyrItZlGpGpmtK258X/Z1qGVU6dJQkrmXjYFs0CCgYFjgQSCU97GdCRrofJ9Z/fUvi+zDEMJnOTcfxi02SY42WVvwIbbK1IJBKSUgEjVIOnqFXnwVQe2L3778qeCqD2xe/fflVJ+i5tWdbzN6y2gsVHwRG0qS8FUHti9++/Ksi27xZuz/ACbZVCtF7uiGMiyhm1Tw7I3yphSFkhJ08U6gcahsf/dXcye0KXBml10rxZoGQ2yRbrpDYuECQnceiyWw424nr0Uk8CKn/BVB7Yvfvvyp4KoPbF799+VNkNf6q7mY1+k+hmXDk/7Mx1YBjY/9rZ/DXPb9h2zy0z406Fg+PxJkZ1LzEhm2tJW04kgpUkhOoIIBBHorSvBVB7Yvfvvyp4KoPbF799+VbNmT67zI65Q5fBFTy/8AMa/t4/3yK2KqQvZLbHSgPXK7yG0rS5zbszVKilQUNRp6QKu9dPJ6CyagqWdd3b70l7ihlVeNeScRSlK3lEUpSgFKUoBSlKAV535V3032Cf56jfdOV6IrzvyrvpvsE/z1G+6coD0RSlKAUpSgFKUoBSlKAUpSgFKUoBSlKAV535V3032Cf56jfdOV6IrzvyrvpvsE/wA9RvunKA9EUpSgFKUoBSlKAUpSgFKUoBSlUi4bS0uuKbsNvN3CToZjjvMxfrSvQlz60pKf7XonGEp7icISm7RVy70rNTmmWqOoh2Vsfolx5Wn7dB/pTpnl36tZP4nq2aJcy7yxqlbgaVXw65U+xR7YLtpvmNBCu9Tiu7bW4ok78Rwko4k6kpIUgk9akGvr30zy79Wsn8T1ZHtu2MHb1f8AD7tkUS0h/HJndCUNFzdltEhSmHdRxQVJSfSBvAabxNNEuZd41StwJXkFbCzsY2GwpM+MWMjyTduc8LTottBH5Bk+cbqDqQeIU4sV6RrNemeXfq1k/iep0zy79Wsn8T1NEuZd41StwNKpWa9M8u/VrJ/E9XMztBv8RW9NscWazrxNulkOgepDiQk/xj4NFwku/wDsPJay/SaJSoywZJAyWIp+C8V82rcdZcSUOsq013VoPFJ048esEEagg1J1qlFxdmiq007MUpSomBSlKAz7Pboq83M462rSA20l247p/nd4+Iwf7JAKljzjdSdUqUK6SUhKQlIAAGgA81dJhancjyhxf84bmUnhx0Sy0lP/ANQP31W9sWXXDAtluUZDaoqZlxtsFyQw0tJUneA8pQHEpT5RA8wNbK+DVNbl5tY/XCx6LJ4xp0k/5LjSvLU7bRkez6Zls8Zm1tItVqxRF0aeZjRmmGprz6W20OFlI4EDfSN4EI39deChYMfybaza7ktdzi3yVZ126W5MmXqBbIyYLyGVLaWwIz7ilJKk7pS4FdYO9wNVrGxVk3azPQtK88YZn2aW5rZFfL7kvfuDmcTSbb+97LKIyzBVJQtpSAF66tkKClEHeJATwAhcH2obWc3i2LLbbarxLt1zlNum1Kh21FsRCU5uq3X+6O6ecS3qreUnQqTpuAHgsNMuD+v/AE9Q0rEMCyTKLzJz7Ib9mve7Hsdvl0htREwWOaTGZSdFvL3N8hvUEbpSTueMVb3Cv7NNouZ3TPoOP3C9XuZaMjssuXb7tdbLEgPNOtlvdejoQVaoKXQd19GuoT1gkUsZ0qww3no1DiXU7yFBadSNUnUag6Gv1XlHZrld+2U8k2xXqHc5V9n3RcWBbIUiPH5uC47JU3qjdDZX5W9o6vQlKRvJBJrTNkt22jqzF+FkkO8ycdXBU6m4X2Jb4z7UoLSA2kRHlhSFJUo+MkEFA4nWlhGqpWw3mrynpNofTeLchS5sZOqmEK0EpoaktK8x6yUk+SrQ9RUDqNunsXW3xpsVwPRZLSXmnE9SkKAKT+0EVnNT+yZalYHASfJadkst/ZokOJRp6t1KatL7VK76Gl33/rxOdl0ErTRb6UpWo5IpSlAZplEBVizB6QQRCvAStKyfFTJQgJKPrUhKSPTuL/bXs+tkm9YRfoENt96XKhPMtNxZfcjqlKQQAl7RXNnj5Wh09BrYLtaYl8tz0GcyH4rwAUgkg6gghQI0KVAgEKBBBAIIIBqgT8XyCxLKY7QyCECAhaFoalJH9sKIQs+sFOv6Pp3OOms08fM6+TZTHM0c3Y84bI9lOTtSLpYcgtEyBs9nW12NLs96kW55bz6ikJUyYTTYQkICwSo66lOgBGo0rFtjTGMRpUVWV5PeYbsFduai3Sel1uOyrQeIAhOqgAAFL3iBqNeJ1uZnXFJ0XjV6SodY7mSr/wAhRFO+E/8Aq5evdPnUdXq8PIuRdGK+9f8Akq8TZFZ4VuwGEiTOLWFhAt5U4jed3Yyo457xPG8RZPi7vHTzcKjcY2FWzDLw1Is2QZFAs7MlctrHGp4FubWokqCUbm/uFSirc393U9VXrvhP/q5evdPnULiW0OHnlo76Y/brndrdzzjHdMaNqgrQopWAdeOhBFNXq8Line truncated
+      "text/plain": [
+       "<IPython.core.display.Image object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": ["from IPython.display import Image, display\n\ntry:\n    display(Image(graph.get_graph(xray=True).draw_mermaid_png()))\nexcept Exception:\n    # This requires some extra dependencies and is optional\n    pass"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "242aa2f0-2c31-462f-a958-ff9ae0cf7c62",
+   "metadata": {},
+   "outputs": [],
+   "source": ["_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n    \"configurable\": {\n        # Checkpoints are accessed by thread_id\n        \"thread_id\": thread_id,\n    }\n}\n\nquestion = \"Write a Python program that prints 'Hello, World!' to the console.\"\nevents = graph.stream(\n    {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n    _print_event(event, _printed)"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6924e707-5970-4254-a748-fa75628916f2",
+   "metadata": {},
+   "source": [
+    "`Trace:`\n",
+    "\n",
+    "https://smith.langchain.com/public/53bcdaab-e3c5-4423-9908-c44595325c38/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "390b2768-f395-4aea-8b0e-9d36212a31ac",
+   "metadata": {},
+   "outputs": [],
+   "source": ["_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n    \"configurable\": {\n        # Checkpoints are accessed by thread_id\n        \"thread_id\": thread_id,\n    }\n}\n\nquestion = \"\"\"Create a Python program that checks if a given string is a palindrome. A palindrome is a word, phrase, number, or other sequence of characters that reads the same forward and backward (ignoring spaces, punctuation, and capitalization).\n\nRequirements:\nThe program should define a function is_palindrome(s) that takes a string s as input.\nThe function should return True if the string is a palindrome and False otherwise.\nIgnore spaces, punctuation, and case differences when checking for palindromes.\n\nGive an example of it working on an example input word.\"\"\"\n\nevents = graph.stream(\n    {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n    _print_event(event, _printed)"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f96e0137-6df3-4a2a-8711-c9cb7dc66831",
+   "metadata": {},
+   "source": [
+    "Trace:\n",
+    "\n",
+    "https://smith.langchain.com/public/e749936d-7746-49de-b980-c41b17986e79/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a3f946b-e2f2-44d9-905b-09f36980cf9f",
+   "metadata": {},
+   "outputs": [],
+   "source": ["_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n    \"configurable\": {\n        # Checkpoints are accessed by thread_id\n        \"thread_id\": thread_id,\n    }\n}\n\nquestion = \"\"\"Write a program that prints the numbers from 1 to 100. \nBut for multiples of three, print \"Fizz\" instead of the number, and for the multiples of five, print \"Buzz\". \nFor numbers which are multiples of both three and five, print \"FizzBuzz\".\"\"\"\n\nevents = graph.stream(\n    {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n    _print_event(event, _printed)"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8f3ef03b-0a07-49f5-9cbf-15e2503d020e",
+   "metadata": {},
+   "source": [
+    "Trace: \n",
+    "\n",
+    "https://smith.langchain.com/public/f5c19708-7592-4512-9f00-9696ab34a9eb/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2bb883df-540b-46ab-9415-fe27db68456f",
+   "metadata": {},
+   "outputs": [],
+   "source": ["import uuid\n\n_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n    \"configurable\": {\n        # Checkpoints are accessed by thread_id\n        \"thread_id\": thread_id,\n    }\n}\n\nquestion = \"\"\"I want to vectorize a function\n\n        frame = np.zeros((out_h, out_w, 3), dtype=np.uint8)\n        for i, val1 in enumerate(rows):\n            for j, val2 in enumerate(cols):\n                for j, val3 in enumerate(ch):\n                    # Assuming you want to store the pair as tuples in the matrix\n                    frame[i, j, k] = image[val1, val2, val3]\n\n        out.write(np.array(frame))\n\nwith a simple numpy function that does something like this what is it called. Show me a test case with this working.\"\"\"\n\nevents = graph.stream(\n    {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n    _print_event(event, _printed)"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "750a3292-1e0e-49cf-8b28-bef179afe6a2",
+   "metadata": {},
+   "source": [
+    "Trace w/ good example of self-correction:\n",
+    "\n",
+    "https://smith.langchain.com/public/b54778a0-d267-4f09-bc28-71761201c522/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ee05da1f-c272-405d-8a7b-552cfc3106e1",
+   "metadata": {},
+   "outputs": [],
+   "source": ["_printed = set()\nthread_id = str(uuid.uuid4())\nconfig = {\n    \"configurable\": {\n        # Checkpoints are accessed by thread_id\n        \"thread_id\": thread_id,\n    }\n}\n\nquestion = \"\"\"Create a Python program that allows two players to play a game of Tic-Tac-Toe. The game should be played on a 3x3 grid. The program should:\n\n- Allow players to take turns to input their moves.\n- Check for invalid moves (e.g., placing a marker on an already occupied space).\n- Determine and announce the winner or if the game ends in a draw.\n\nRequirements:\n- Use a 2D list to represent the Tic-Tac-Toe board.\n- Use functions to modularize the code.\n- Validate player input.\n- Check for win conditions and draw conditions after each move.\"\"\"\n\nevents = graph.stream(\n    {\"messages\": [(\"user\", question)], \"iterations\": 0}, config, stream_mode=\"values\"\n)\nfor event in events:\n    _print_event(event, _printed)"]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3d900cd6-2df9-467d-8e74-803527269008",
+   "metadata": {},
+   "source": [
+    "Trace w/ good example of failure to correct:\n",
+    "\n",
+    "https://smith.langchain.com/public/871ae736-2f77-44d4-b0da-a600d8f5377d/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "814fc2a4-8e5b-4faa-8f52-3977226bd09a",
+   "metadata": {},
+   "outputs": [],
+   "source": [""]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/configuration.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e9a58c69",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/configuration.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/create-react-agent-hitl.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a1e6efeb",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/create-react-agent-hitl.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/create-react-agent-memory.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "1ef41a89",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/create-react-agent-memory.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/create-react-agent-system-prompt.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9e2f7902",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/create-react-agent-system-prompt.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/create-react-agent.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "eb07372e",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/create-react-agent.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/customer-support/customer-support.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "a8232bc9",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/customer-support/customer-support.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/extraction/retries.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8dbdba5b",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/extraction/retries.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/human_in_the_loop/wait-user-input.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "3ecab357",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/human_in_the_loop/wait-user-input.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/input_output_schema.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "fc0793cb",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/input_output_schema.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/lats/lats.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "09038b53",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/lats/lats.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/llm-compiler/LLMCompiler.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "85205e97",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/llm-compiler/LLMCompiler.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/map-reduce.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "42abb708",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/map-reduce.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/memory/add-summary-conversation-history.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "298784f6",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/memory/add-summary-conversation-history.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/memory/delete-messages.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "3f4370fd",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/memory/delete-messages.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/memory/manage-conversation-history.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "6ec7cb13",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/memory/manage-conversation-history.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/multi_agent/hierarchical_agent_teams.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "5cc8a2ad",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/multi_agent/hierarchical_agent_teams.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/multi_agent/multi-agent-collaboration.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d2b507b9",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/multi_agent/multi-agent-collaboration.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/node-retries.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "017a01f4",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/node-retries.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "env",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

examples/pass-config-to-tools.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "05f6ad0a",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/pass-config-to-tools.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/pass-run-time-values-to-tools.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8f38bec5",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/pass-run-time-values-to-tools.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/pass_private_state.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4da17088",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/pass_private_state.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/persistence.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d16e8b9c",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/memory/add-memory.md."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/persistence_mongodb.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "78217098",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/persistence_mongodb.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/persistence_postgres.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "18526f23",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/memory/add-memory.md"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/persistence_redis.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "eee6ecdd",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/persistence_redis.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/plan-and-execute/plan-and-execute.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9138f92e",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/plan-and-execute/plan-and-execute.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_adaptive_rag.ipynb

@@ -0,0 +1,930 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "36fa621a-9d3d-4860-a17c-5d20e6987481.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABY8AAALqCAYAAAB9kqsRAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAWPoAMABAAAAAEAAALqAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdDdDgckAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjc0NjwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNDIzPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "5afcaed0-3d55-4e1f-95d3-c32c751c29d8",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "# Adaptive RAG\n",
+    "\n",
+    "Adaptive RAG is a strategy for RAG that unites (1) [query analysis](https://blog.langchain.dev/query-construction/) with (2) [active / self-corrective RAG](https://blog.langchain.dev/agentic-rag-with-langgraph/).\n",
+    "\n",
+    "In the [paper](https://arxiv.org/abs/2403.14403), they report query analysis to route across:\n",
+    "\n",
+    "* No Retrieval\n",
+    "* Single-shot RAG\n",
+    "* Iterative RAG\n",
+    "\n",
+    "Let's build on this using LangGraph. \n",
+    "\n",
+    "In our implementation, we will route between:\n",
+    "\n",
+    "* Web search: for questions related to recent events\n",
+    "* Self-corrective RAG: for questions related to our index\n",
+    "\n",
+    "![Screenshot 2024-03-26 at 1.36.03 PM.png](attachment:36fa621a-9d3d-4860-a17c-5d20e6987481.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a85501ca-eb89-4795-aeab-cdab050ead6b",
+   "metadata": {},
+   "source": [
+    "## Setup\n",
+    "\n",
+    "First, let's install our required packages and set our API keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "53d1a740-9fea-4a6e-8f95-fb9dbf1c80a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture --no-stderr\n",
+    "! pip install -U langchain_community tiktoken langchain-openai langchain-cohere langchainhub chromadb langchain langgraph  tavily-python"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "222f204d-956f-4128-b597-2c698120edda",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "\n",
+    "def _set_env(var: str):\n",
+    "    if not os.environ.get(var):\n",
+    "        os.environ[var] = getpass.getpass(f\"{var}: \")\n",
+    "\n",
+    "\n",
+    "_set_env(\"OPENAI_API_KEY\")\n",
+    "_set_env(\"COHERE_API_KEY\")\n",
+    "_set_env(\"TAVILY_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "47e04b18",
+   "metadata": {},
+   "source": [
+    "<div class=\"admonition tip\">\n",
+    "    <p class=\"admonition-title\">Set up <a href=\"https://smith.langchain.com\">LangSmith</a> for LangGraph development</p>\n",
+    "    <p style=\"padding-top: 5px;\">\n",
+    "        Sign up for LangSmith to quickly spot issues and improve the performance of your LangGraph projects. LangSmith lets you use trace data to debug, test, and monitor your LLM apps built with LangGraph — read more about how to get started <a href=\"https://docs.smith.langchain.com\">here</a>. \n",
+    "    </p>\n",
+    "</div>    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9ac1c2cd-81fb-40eb-8ba1-e9197800cba6",
+   "metadata": {},
+   "source": [
+    "## Create Index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "b224e5ba-50ca-495a-a7fa-0f75a080e03c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Build Index\n",
+    "\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_community.document_loaders import WebBaseLoader\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "\n",
+    "### from langchain_cohere import CohereEmbeddings\n",
+    "\n",
+    "# Set embeddings\n",
+    "embd = OpenAIEmbeddings()\n",
+    "\n",
+    "# Docs to index\n",
+    "urls = [\n",
+    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+    "]\n",
+    "\n",
+    "# Load\n",
+    "docs = [WebBaseLoader(url).load() for url in urls]\n",
+    "docs_list = [item for sublist in docs for item in sublist]\n",
+    "\n",
+    "# Split\n",
+    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+    "    chunk_size=500, chunk_overlap=0\n",
+    ")\n",
+    "doc_splits = text_splitter.split_documents(docs_list)\n",
+    "\n",
+    "# Add to vectorstore\n",
+    "vectorstore = Chroma.from_documents(\n",
+    "    documents=doc_splits,\n",
+    "    collection_name=\"rag-chroma\",\n",
+    "    embedding=embd,\n",
+    ")\n",
+    "retriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0f52b427-750c-40f8-8893-e9caab3afd8d",
+   "metadata": {},
+   "source": [
+    "## LLMs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "4dec9d98-f3dc-4b7f-abc0-9d01c754f2be",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "datasource='web_search'\n",
+      "datasource='vectorstore'\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Router\n",
+    "\n",
+    "from typing import Literal\n",
+    "\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class RouteQuery(BaseModel):\n",
+    "    \"\"\"Route a user query to the most relevant datasource.\"\"\"\n",
+    "\n",
+    "    datasource: Literal[\"vectorstore\", \"web_search\"] = Field(\n",
+    "        ...,\n",
+    "        description=\"Given a user question choose to route it to web search or a vectorstore.\",\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_router = llm.with_structured_output(RouteQuery)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You are an expert at routing a user question to a vectorstore or web search.\n",
+    "The vectorstore contains documents related to agents, prompt engineering, and adversarial attacks.\n",
+    "Use the vectorstore for questions on these topics. Otherwise, use web-search.\"\"\"\n",
+    "route_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\"human\", \"{question}\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "question_router = route_prompt | structured_llm_router\n",
+    "print(\n",
+    "    question_router.invoke(\n",
+    "        {\"question\": \"Who will the Bears draft first in the NFL draft?\"}\n",
+    "    )\n",
+    ")\n",
+    "print(question_router.invoke({\"question\": \"What are the types of agent memory?\"}))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "856801cb-f42a-44e7-956f-47845e3664ca",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "binary_score='no'\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Retrieval Grader\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeDocuments(BaseModel):\n",
+    "    \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+    "    If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
+    "    It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
+    "    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
+    "grade_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "retrieval_grader = grade_prompt | structured_llm_grader\n",
+    "question = \"agent memory\"\n",
+    "docs = retriever.get_relevant_documents(question)\n",
+    "doc_txt = docs[1].page_content\n",
+    "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "2272333e-50b2-42ab-b472-e1055a3b94a8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The design of generative agents combines LLM with memory, planning, and reflection mechanisms to enable agents to behave based on past experience and interact with other agents. Memory stream is a long-term memory module that records agents' experiences in natural language. The retrieval model surfaces context to inform the agent's behavior based on relevance, recency, and importance.\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Generate\n",
+    "\n",
+    "from langchain import hub\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
+    "\n",
+    "\n",
+    "# Post-processing\n",
+    "def format_docs(docs):\n",
+    "    return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+    "\n",
+    "\n",
+    "# Chain\n",
+    "rag_chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "# Run\n",
+    "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+    "print(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "f0c08d14-77a0-4eed-b882-2d636abb22a3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GradeHallucinations(binary_score='yes')"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Hallucination Grader\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeHallucinations(BaseModel):\n",
+    "    \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeHallucinations)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n \n",
+    "     Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\n",
+    "hallucination_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "hallucination_grader = hallucination_prompt | structured_llm_grader\n",
+    "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ded99680-437a-4c9d-b860-619c88949d84",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GradeAnswer(binary_score='yes')"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Answer Grader\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeAnswer(BaseModel):\n",
+    "    \"\"\"Binary score to assess answer addresses question.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Answer addresses the question, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeAnswer)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n \n",
+    "     Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\n",
+    "answer_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "answer_grader = answer_prompt | structured_llm_grader\n",
+    "answer_grader.invoke({\"question\": question, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "9d75f1d7-a47a-4577-bb0d-84b504b0867e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"What is the role of memory in an agent's functioning?\""
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Question Re-writer\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+    "     for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
+    "re_write_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\n",
+    "            \"human\",\n",
+    "            \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
+    "        ),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+    "question_rewriter.invoke({\"question\": question})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d07c0b31-b919-4498-869f-9673125c2473",
+   "metadata": {},
+   "source": [
+    "## Web Search Tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "01d829bb-1074-4976-b650-ead41dcb9788",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Search\n",
+    "\n",
+    "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+    "\n",
+    "web_search_tool = TavilySearchResults(k=3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "efbbff0e-8843-45bb-b2ff-137bef707ef4",
+   "metadata": {},
+   "source": [
+    "## Construct the Graph \n",
+    "\n",
+    "Capture the flow in as a graph.\n",
+    "\n",
+    "### Define Graph State"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "e723fcdb-06e6-402d-912e-899795b78408",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List\n",
+    "\n",
+    "from typing_extensions import TypedDict\n",
+    "\n",
+    "\n",
+    "class GraphState(TypedDict):\n",
+    "    \"\"\"\n",
+    "    Represents the state of our graph.\n",
+    "\n",
+    "    Attributes:\n",
+    "        question: question\n",
+    "        generation: LLM generation\n",
+    "        documents: list of documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question: str\n",
+    "    generation: str\n",
+    "    documents: List[str]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e2d6c0d-42e8-4399-9751-e315be16607a",
+   "metadata": {},
+   "source": [
+    "### Define Graph Flow "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "b76b5ec3-0720-443d-85b1-c0e79659ca0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.schema import Document\n",
+    "\n",
+    "\n",
+    "def retrieve(state):\n",
+    "    \"\"\"\n",
+    "    Retrieve documents\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, documents, that contains retrieved documents\n",
+    "    \"\"\"\n",
+    "    print(\"---RETRIEVE---\")\n",
+    "    question = state[\"question\"]\n",
+    "\n",
+    "    # Retrieval\n",
+    "    documents = retriever.invoke(question)\n",
+    "    return {\"documents\": documents, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def generate(state):\n",
+    "    \"\"\"\n",
+    "    Generate answer\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, generation, that contains LLM generation\n",
+    "    \"\"\"\n",
+    "    print(\"---GENERATE---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # RAG generation\n",
+    "    generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+    "\n",
+    "\n",
+    "def grade_documents(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the retrieved documents are relevant to the question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with only filtered relevant documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Score each doc\n",
+    "    filtered_docs = []\n",
+    "    for d in documents:\n",
+    "        score = retrieval_grader.invoke(\n",
+    "            {\"question\": question, \"document\": d.page_content}\n",
+    "        )\n",
+    "        grade = score.binary_score\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+    "            filtered_docs.append(d)\n",
+    "        else:\n",
+    "            print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+    "            continue\n",
+    "    return {\"documents\": filtered_docs, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def transform_query(state):\n",
+    "    \"\"\"\n",
+    "    Transform the query to produce a better question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates question key with a re-phrased question\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---TRANSFORM QUERY---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Re-write question\n",
+    "    better_question = question_rewriter.invoke({\"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": better_question}\n",
+    "\n",
+    "\n",
+    "def web_search(state):\n",
+    "    \"\"\"\n",
+    "    Web search based on the re-phrased question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with appended web results\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---WEB SEARCH---\")\n",
+    "    question = state[\"question\"]\n",
+    "\n",
+    "    # Web search\n",
+    "    docs = web_search_tool.invoke({\"query\": question})\n",
+    "    web_results = \"\\n\".join([d[\"content\"] for d in docs])\n",
+    "    web_results = Document(page_content=web_results)\n",
+    "\n",
+    "    return {\"documents\": web_results, \"question\": question}\n",
+    "\n",
+    "\n",
+    "### Edges ###\n",
+    "\n",
+    "\n",
+    "def route_question(state):\n",
+    "    \"\"\"\n",
+    "    Route question to web search or RAG.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---ROUTE QUESTION---\")\n",
+    "    question = state[\"question\"]\n",
+    "    source = question_router.invoke({\"question\": question})\n",
+    "    if source.datasource == \"web_search\":\n",
+    "        print(\"---ROUTE QUESTION TO WEB SEARCH---\")\n",
+    "        return \"web_search\"\n",
+    "    elif source.datasource == \"vectorstore\":\n",
+    "        print(\"---ROUTE QUESTION TO RAG---\")\n",
+    "        return \"vectorstore\"\n",
+    "\n",
+    "\n",
+    "def decide_to_generate(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether to generate an answer, or re-generate a question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Binary decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+    "    state[\"question\"]\n",
+    "    filtered_documents = state[\"documents\"]\n",
+    "\n",
+    "    if not filtered_documents:\n",
+    "        # All documents have been filtered check_relevance\n",
+    "        # We will re-generate a new query\n",
+    "        print(\n",
+    "            \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+    "        )\n",
+    "        return \"transform_query\"\n",
+    "    else:\n",
+    "        # We have relevant documents, so generate answer\n",
+    "        print(\"---DECISION: GENERATE---\")\n",
+    "        return \"generate\"\n",
+    "\n",
+    "\n",
+    "def grade_generation_v_documents_and_question(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the generation is grounded in the document and answers question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK HALLUCINATIONS---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "    generation = state[\"generation\"]\n",
+    "\n",
+    "    score = hallucination_grader.invoke(\n",
+    "        {\"documents\": documents, \"generation\": generation}\n",
+    "    )\n",
+    "    grade = score.binary_score\n",
+    "\n",
+    "    # Check hallucination\n",
+    "    if grade == \"yes\":\n",
+    "        print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
+    "        # Check question-answering\n",
+    "        print(\"---GRADE GENERATION vs QUESTION---\")\n",
+    "        score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
+    "        grade = score.binary_score\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
+    "            return \"useful\"\n",
+    "        else:\n",
+    "            print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
+    "            return \"not useful\"\n",
+    "    else:\n",
+    "        pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
+    "        return \"not supported\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3ab01f36-5628-49ab-bfd3-84bb6f1a1b0f",
+   "metadata": {},
+   "source": [
+    "### Compile Graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "67854e07-9293-4c3c-bf9a-bc9a605570ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langgraph.graph import END, StateGraph, START\n",
+    "\n",
+    "workflow = StateGraph(GraphState)\n",
+    "\n",
+    "# Define the nodes\n",
+    "workflow.add_node(\"web_search\", web_search)  # web search\n",
+    "workflow.add_node(\"retrieve\", retrieve)  # retrieve\n",
+    "workflow.add_node(\"grade_documents\", grade_documents)  # grade documents\n",
+    "workflow.add_node(\"generate\", generate)  # generate\n",
+    "workflow.add_node(\"transform_query\", transform_query)  # transform_query\n",
+    "\n",
+    "# Build graph\n",
+    "workflow.add_conditional_edges(\n",
+    "    START,\n",
+    "    route_question,\n",
+    "    {\n",
+    "        \"web_search\": \"web_search\",\n",
+    "        \"vectorstore\": \"retrieve\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"web_search\", \"generate\")\n",
+    "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"grade_documents\",\n",
+    "    decide_to_generate,\n",
+    "    {\n",
+    "        \"transform_query\": \"transform_query\",\n",
+    "        \"generate\": \"generate\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"transform_query\", \"retrieve\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"generate\",\n",
+    "    grade_generation_v_documents_and_question,\n",
+    "    {\n",
+    "        \"not supported\": \"generate\",\n",
+    "        \"useful\": END,\n",
+    "        \"not useful\": \"transform_query\",\n",
+    "    },\n",
+    ")\n",
+    "\n",
+    "# Compile\n",
+    "app = workflow.compile()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "85bce541",
+   "metadata": {},
+   "source": [
+    "## Use Graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "29acc541-d726-4b75-84d1-a215845fe88a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---ROUTE QUESTION---\n",
+      "---ROUTE QUESTION TO WEB SEARCH---\n",
+      "---WEB SEARCH---\n",
+      "\"Node 'web_search':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "('It is expected that the Chicago Bears could have the opportunity to draft '\n",
+      " 'the first defensive player in the 2024 NFL draft. The Bears have the first '\n",
+      " 'overall pick in the draft, giving them a prime position to select top '\n",
+      " 'talent. The top wide receiver Marvin Harrison Jr. from Ohio State is also '\n",
+      " 'mentioned as a potential pick for the Cardinals.')\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Run\n",
+    "inputs = {\n",
+    "    \"question\": \"What player at the Bears expected to draft first in the 2024 NFL draft?\"\n",
+    "}\n",
+    "for output in app.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        # Node\n",
+    "        pprint(f\"Node '{key}':\")\n",
+    "        # Optional: print full state at each node\n",
+    "        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+    "    pprint(\"\\n---\\n\")\n",
+    "\n",
+    "# Final generation\n",
+    "pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11fddd00-58bf-4910-bf36-be9e5bfba778",
+   "metadata": {},
+   "source": [
+    "Trace: \n",
+    "\n",
+    "https://smith.langchain.com/public/7e3aa7e5-c51f-45c2-bc66-b34f17ff2263/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "69a985dd-03c6-45af-a67b-b15746a2cb5f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---ROUTE QUESTION---\n",
+      "---ROUTE QUESTION TO RAG---\n",
+      "---RETRIEVE---\n",
+      "\"Node 'retrieve':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---ASSESS GRADED DOCUMENTS---\n",
+      "---DECISION: GENERATE---\n",
+      "\"Node 'grade_documents':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "('The types of agent memory include Sensory Memory, Short-Term Memory (STM) or '\n",
+      " 'Working Memory, and Long-Term Memory (LTM) with subtypes of Explicit / '\n",
+      " 'declarative memory and Implicit / procedural memory. Sensory memory retains '\n",
+      " 'sensory information briefly, STM stores information for cognitive tasks, and '\n",
+      " 'LTM stores information for a long time with different types of memories.')\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run\n",
+    "inputs = {\"question\": \"What are the types of agent memory?\"}\n",
+    "for output in app.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        # Node\n",
+    "        pprint(f\"Node '{key}':\")\n",
+    "        # Optional: print full state at each node\n",
+    "        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+    "    pprint(\"\\n---\\n\")\n",
+    "\n",
+    "# Final generation\n",
+    "pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ebf41097-fc4c-4072-95b3-e7e07731ada1",
+   "metadata": {},
+   "source": [
+    "Trace: \n",
+    "\n",
+    "https://smith.langchain.com/public/fdf0a180-6d15-4d09-bb92-f84f2105ca51/r"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_adaptive_rag_cohere.ipynb

@@ -0,0 +1,597 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "2a4ecdd2-280d-4311-a2cd-cd6138090be9.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABpMAAALQCAYAAAB8A4i9AAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAaToAMABAAAAAEAAALQAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdLA+VsAAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjcyMDwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNjgzPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "5afcaed0-3d55-4e1f-95d3-c32c751c29d8",
+   "metadata": {
+    "id": "5afcaed0-3d55-4e1f-95d3-c32c751c29d8"
+   },
+   "source": [
+    "# Adaptive RAG Cohere Command R\n",
+    "\n",
+    "Adaptive RAG is a strategy for RAG that unites (1) [query analysis](https://blog.langchain.dev/query-construction/) with (2) [active / self-corrective RAG](https://blog.langchain.dev/agentic-rag-with-langgraph/).\n",
+    "\n",
+    "In the paper, they report query analysis to route across:\n",
+    "\n",
+    "* No Retrieval (LLM answers)\n",
+    "* Single-shot RAG\n",
+    "* Iterative RAG\n",
+    "\n",
+    "Let's build on this to perform query analysis to route across some more interesting cases:\n",
+    "\n",
+    "* No Retrieval (LLM answers)\n",
+    "* Web-search\n",
+    "* Iterative RAG\n",
+    "\n",
+    "We'll use [Command R](https://cohere.com/blog/command-r), a recent release from Cohere that:\n",
+    "\n",
+    "* Has strong accuracy on RAG and Tool Use\n",
+    "* Has 128k context\n",
+    "* Has low latency \n",
+    "  \n",
+    "![Screenshot 2024-04-02 at 8.11.18 PM.png](attachment:2a4ecdd2-280d-4311-a2cd-cd6138090be9.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a85501ca-eb89-4795-aeab-cdab050ead6b",
+   "metadata": {
+    "id": "a85501ca-eb89-4795-aeab-cdab050ead6b"
+   },
+   "source": [
+    "# Environment"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f6c329ba-cb85-4576-9828-4f2ac648d1a6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install --quiet langchain langchain_cohere langchain-openai tiktoken langchainhub chromadb langgraph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "222f204d-956f-4128-b597-2c698120edda",
+   "metadata": {
+    "id": "222f204d-956f-4128-b597-2c698120edda"
+   },
+   "outputs": [],
+   "source": [
+    "### LLMs\nimport os\n\nos.environ[\"COHERE_API_KEY\"] = \"<your-api-key>\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "08edba00-988a-478b-96fc-ae0199cbef49",
+   "metadata": {
+    "id": "08edba00-988a-478b-96fc-ae0199cbef49"
+   },
+   "outputs": [],
+   "source": [
+    "# ### Tracing (optional)\n# os.environ['LANGCHAIN_TRACING_V2'] = 'true'\n# os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'\n# os.environ['LANGCHAIN_API_KEY'] ='<your-api-key>'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9ac1c2cd-81fb-40eb-8ba1-e9197800cba6",
+   "metadata": {
+    "id": "9ac1c2cd-81fb-40eb-8ba1-e9197800cba6"
+   },
+   "source": [
+    "## Index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "b224e5ba-50ca-495a-a7fa-0f75a080e03c",
+   "metadata": {
+    "id": "b224e5ba-50ca-495a-a7fa-0f75a080e03c"
+   },
+   "outputs": [],
+   "source": [
+    "### Build Index\n\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_cohere import CohereEmbeddings\nfrom langchain_community.document_loaders import WebBaseLoader\nfrom langchain_community.vectorstores import Chroma\n\n# Set embeddings\nembd = CohereEmbeddings()\n\n# Docs to index\nurls = [\n    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n    \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n    \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n]\n\n# Load\ndocs = [WebBaseLoader(url).load() for url in urls]\ndocs_list = [item for sublist in docs for item in sublist]\n\n# Split\ntext_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n    chunk_size=512, chunk_overlap=0\n)\ndoc_splits = text_splitter.split_documents(docs_list)\n\n# Add to vectorstore\nvectorstore = Chroma.from_documents(\n    documents=doc_splits,\n    embedding=embd,\n)\n\nretriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0f52b427-750c-40f8-8893-e9caab3afd8d",
+   "metadata": {
+    "id": "0f52b427-750c-40f8-8893-e9caab3afd8d"
+   },
+   "source": [
+    "## LLMs"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "H5CztTqsBOTZ",
+   "metadata": {
+    "id": "H5CztTqsBOTZ"
+   },
+   "source": [
+    "We use a router to pick between tools. \n",
+    " \n",
+    "Cohere model decides which tool(s) to call, as well as the how to query them."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "bYK-e0diGdPf",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "bYK-e0diGdPf",
+    "outputId": "895a8ea5-57ee-49fe-ef28-277eac8a7bb7"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'id': 'f811e3b9-052e-49db-a234-5fc3efbcc5ba', 'function': {'name': 'web_search', 'arguments': '{\"query\": \"NFL draft bears first pick\"}'}, 'type': 'function'}]\n",
+      "[{'id': '4bc53113-8f32-4d6d-ac9b-c07ef9aae9fd', 'function': {'name': 'vectorstore', 'arguments': '{\"query\": \"types of agent memory\"}'}, 'type': 'function'}]\n",
+      "False\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Router\n\nfrom langchain_cohere import ChatCohere\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.pydantic_v1 import BaseModel, Field\n\n\n# Data model\nclass web_search(BaseModel):\n    \"\"\"\n    The internet. Use web_search for questions that are related to anything else than agents, prompt engineering, and adversarial attacks.\n    \"\"\"\n\n    query: str = Field(description=\"The query to use when searching the internet.\")\n\n\nclass vectorstore(BaseModel):\n    \"\"\"\n    A vectorstore containing documents related to agents, prompt engineering, and adversarial attacks. Use the vectorstore for questions on these topics.\n    \"\"\"\n\n    query: str = Field(description=\"The query to use when searching the vectorstore.\")\n\n\n# Preamble\npreamble = \"\"\"You are an expert at routing a user question to a vectorstore or web search.\nThe vectorstore contains documents related to agents, prompt engineering, and adversarial attacks.\nUse the vectorstore for questions on these topics. Otherwise, use web-search.\"\"\"\n\n# LLM with tool use and preamble\nllm = ChatCohere(model=\"command-r\", temperature=0)\nstructured_llm_router = llm.bind_tools(\n    tools=[web_search, vectorstore], preamble=preamble\n)\n\n# Prompt\nroute_prompt = ChatPromptTemplate.from_messages(\n    [\n        (\"human\", \"{question}\"),\n    ]\n)\n\nquestion_router = route_prompt | structured_llm_router\nresponse = question_router.invoke(\n    {\"question\": \"Who will the Bears draft first in the NFL draft?\"}\n)\nprint(response.response_metadata[\"tool_calls\"])\nresponse = question_router.invoke({\"question\": \"What are the types of agent memory?\"})\nprint(response.response_metadata[\"tool_calls\"])\nresponse = question_router.invoke({\"question\": \"Hi how are you?\"})\nprint(\"tool_calls\" in response.response_metadata)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "oaLWNbWxBgjE",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "oaLWNbWxBgjE",
+    "outputId": "57a5c27b-044b-4df5-f55d-7bf23d3976d1"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "binary_score='yes'\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Retrieval Grader\n\n\n# Data model\nclass GradeDocuments(BaseModel):\n    \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n\n    binary_score: str = Field(\n        description=\"Documents are relevant to the question, 'yes' or 'no'\"\n    )\n\n\n# Prompt\npreamble = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n\nIf the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\nGive a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n\n# LLM with function call\nllm = ChatCohere(model=\"command-r\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeDocuments, preamble=preamble)\n\ngrade_prompt = ChatPromptTemplate.from_messages(\n    [\n        (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n    ]\n)\n\nretrieval_grader = grade_prompt | structured_llm_grader\nquestion = \"types of agent memory\"\ndocs = retriever.invoke(question)\ndoc_txt = docs[1].page_content\nresponse = retrieval_grader.invoke({\"question\": question, \"document\": doc_txt})\nprint(response)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "D43a7vM4EElX",
+   "metadata": {
+    "id": "D43a7vM4EElX"
+   },
+   "source": [
+    "Generate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "BTIUdjRMEq_h",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "BTIUdjRMEq_h",
+    "outputId": "11a99f62-2449-45db-9281-5bfd60e3c966"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "There are three types of agent memory: sensory memory, short-term memory, and long-term memory.\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Generate\n\nfrom langchain_core.messages import HumanMessage\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Preamble\npreamble = \"\"\"You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise.\"\"\"\n\n# LLM\nllm = ChatCohere(model_name=\"command-r\", temperature=0).bind(preamble=preamble)\n\n\n# Prompt\ndef prompt(x):\n    return ChatPromptTemplate.from_messages(\n        [\n            HumanMessage(\n                f\"Question: {x['question']} \\nAnswer: \",\n                additional_kwargs={\"documents\": x[\"documents\"]},\n            )\n        ]\n    )\n\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"documents\": docs, \"question\": question})\nprint(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "bc000a7d-84b6-4eb2-88ad-65cc62a44431",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "I don't have feelings as an AI chatbot, but I'm here to assist you with any queries or concerns you may have. How can I help you today?\n"
+     ]
+    }
+   ],
+   "source": [
+    "### LLM fallback\n\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Preamble\npreamble = \"\"\"You are an assistant for question-answering tasks. Answer the question based upon your knowledge. Use three sentences maximum and keep the answer concise.\"\"\"\n\n# LLM\nllm = ChatCohere(model_name=\"command-r\", temperature=0).bind(preamble=preamble)\n\n\n# Prompt\ndef prompt(x):\n    return ChatPromptTemplate.from_messages(\n        [HumanMessage(f\"Question: {x['question']} \\nAnswer: \")]\n    )\n\n\n# Chain\nllm_chain = prompt | llm | StrOutputParser()\n\n# Run\nquestion = \"Hi how are you?\"\ngeneration = llm_chain.invoke({\"question\": question})\nprint(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "y0msuR2DHQkY",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "y0msuR2DHQkY",
+    "outputId": "f0e91c2a-5542-45c0-a7a6-60453e0b2bc4"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GradeHallucinations(binary_score='yes')"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Hallucination Grader\n\n\n# Data model\nclass GradeHallucinations(BaseModel):\n    \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n\n    binary_score: str = Field(\n        description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n    )\n\n\n# Preamble\npreamble = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n\nGive a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\n\n# LLM with function call\nllm = ChatCohere(model=\"command-r\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(\n    GradeHallucinations, preamble=preamble\n)\n\n# Prompt\nhallucination_prompt = ChatPromptTemplate.from_messages(\n    [\n        # (\"system\", system),\n        (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n    ]\n)\n\nhallucination_grader = hallucination_prompt | structured_llm_grader\nhallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "f0c08d14-77a0-4eed-b882-2d636abb22a3",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "f0c08d14-77a0-4eed-b882-2d636abb22a3",
+    "outputId": "c4f88c9a-65fd-4dad-e739-3c9bd547a9f5"
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GradeAnswer(binary_score='yes')"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Answer Grader\n\n\n# Data model\nclass GradeAnswer(BaseModel):\n    \"\"\"Binary score to assess answer addresses question.\"\"\"\n\n    binary_score: str = Field(\n        description=\"Answer addresses the question, 'yes' or 'no'\"\n    )\n\n\n# Preamble\npreamble = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n\nGive a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\n\n# LLM with function call\nllm = ChatCohere(model=\"command-r\", temperature=0)\nstructured_llm_grader = llm.with_structured_output(GradeAnswer, preamble=preamble)\n\n# Prompt\nanswer_prompt = ChatPromptTemplate.from_messages(\n    [\n        (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n    ]\n)\n\nanswer_grader = answer_prompt | structured_llm_grader\nanswer_grader.invoke({\"question\": question, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d07c0b31-b919-4498-869f-9673125c2473",
+   "metadata": {
+    "id": "d07c0b31-b919-4498-869f-9673125c2473"
+   },
+   "source": [
+    "## Web Search Tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "01d829bb-1074-4976-b650-ead41dcb9788",
+   "metadata": {
+    "id": "01d829bb-1074-4976-b650-ead41dcb9788"
+   },
+   "outputs": [],
+   "source": [
+    "### Search\n# os.environ['TAVILY_API_KEY'] ='<your-api-key>'\n\nfrom langchain_community.tools.tavily_search import TavilySearchResults\n\nweb_search_tool = TavilySearchResults()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "efbbff0e-8843-45bb-b2ff-137bef707ef4",
+   "metadata": {
+    "id": "efbbff0e-8843-45bb-b2ff-137bef707ef4"
+   },
+   "source": [
+    "# Graph\n",
+    "\n",
+    "Capture the flow in as a graph.\n",
+    "\n",
+    "## Graph state"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "e723fcdb-06e6-402d-912e-899795b78408",
+   "metadata": {
+    "id": "e723fcdb-06e6-402d-912e-899795b78408"
+   },
+   "outputs": [],
+   "source": [
+    "from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n    \"\"\"|\n    Represents the state of our graph.\n\n    Attributes:\n        question: question\n        generation: LLM generation\n        documents: list of documents\n    \"\"\"\n\n    question: str\n    generation: str\n    documents: List[str]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7e2d6c0d-42e8-4399-9751-e315be16607a",
+   "metadata": {
+    "id": "7e2d6c0d-42e8-4399-9751-e315be16607a"
+   },
+   "source": [
+    "## Graph Flow"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "b76b5ec3-0720-443d-85b1-c0e79659ca0a",
+   "metadata": {
+    "id": "b76b5ec3-0720-443d-85b1-c0e79659ca0a"
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.schema import Document\n\n\ndef retrieve(state):\n    \"\"\"\n    Retrieve documents\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): New key added to state, documents, that contains retrieved documents\n    \"\"\"\n    print(\"---RETRIEVE---\")\n    question = state[\"question\"]\n\n    # Retrieval\n    documents = retriever.invoke(question)\n    return {\"documents\": documents, \"question\": question}\n\n\ndef llm_fallback(state):\n    \"\"\"\n    Generate answer using the LLM w/o vectorstore\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): New key added to state, generation, that contains LLM generation\n    \"\"\"\n    print(\"---LLM Fallback---\")\n    question = state[\"question\"]\n    generation = llm_chain.invoke({\"question\": question})\n    return {\"question\": question, \"generation\": generation}\n\n\ndef generate(state):\n    \"\"\"\n    Generate answer using the vectorstore\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): New key added to state, generation, that contains LLM generation\n    \"\"\"\n    print(\"---GENERATE---\")\n    question = state[\"question\"]\n    documents = state[\"documents\"]\n    if not isinstance(documents, list):\n        documents = [documents]\n\n    # RAG generation\n    generation = rag_chain.invoke({\"documents\": documents, \"question\": question})\n    return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n    \"\"\"\n    Determines whether the retrieved documents are relevant to the question.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): Updates documents key with only filtered relevant documents\n    \"\"\"\n\n    print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n    question = state[\"question\"]\n    documents = state[\"documents\"]\n\n    # Score each doc\n    filtered_docs = []\n    for d in documents:\n        score = retrieval_grader.invoke(\n            {\"question\": question, \"document\": d.page_content}\n        )\n        grade = score.binary_score\n        if grade == \"yes\":\n            print(\"---GRADE: DOCUMENT RELEVANT---\")\n            filtered_docs.append(d)\n        else:\n            print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n            continue\n    return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef web_search(state):\n    \"\"\"\n    Web search based on the re-phrased question.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): Updates documents key with appended web results\n    \"\"\"\n\n    print(\"---WEB SEARCH---\")\n    question = state[\"question\"]\n\n    # Web search\n    docs = web_search_tool.invoke({\"query\": question})\n    web_results = \"\\n\".join([d[\"content\"] for d in docs])\n    web_results = Document(page_content=web_results)\n\n    return {\"documents\": web_results, \"question\": question}\n\n\n### Edges ###\n\n\ndef route_question(state):\n    \"\"\"\n    Route question to web search or RAG.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        str: Next node to call\n    \"\"\"\n\n    print(\"---ROUTE QUESTION---\")\n    question = state[\"question\"]\n    source = question_router.invoke({\"question\": question})\n\n    # Fallback to LLM or raise error if no decision\n    if \"tool_calls\" not in source.additional_kwargs:\n        print(\"---ROUTE QUESTION TO LLM---\")\n        return \"llm_fallback\"\n    if len(source.additional_kwargs[\"tool_calls\"]) == 0:\n        raise \"Router could not decide source\"\n\n    # Choose datasource\n    datasource = source.additional_kwargs[\"tool_calls\"][0][\"function\"][\"name\"]\n    if datasource == \"web_search\":\n        print(\"---ROUTE QUESTION TO WEB SEARCH---\")\n        return \"web_search\"\n    elif datasource == \"vectorstore\":\n        print(\"---ROUTE QUESTION TO RAG---\")\n        return \"vectorstore\"\n    else:\n        print(\"---ROUTE QUESTION TO LLM---\")\n        return \"vectorstore\"\n\n\ndef decide_to_generate(state):\n    \"\"\"\n    Determines whether to generate an answer, or re-generate a question.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        str: Binary decision for next node to call\n    \"\"\"\n\n    print(\"---ASSESS GRADED DOCUMENTS---\")\n    state[\"question\"]\n    filtered_documents = state[\"documents\"]\n\n    if not filtered_documents:\n        # All documents have been filtered check_relevance\n        # We will re-generate a new query\n        print(\"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, WEB SEARCH---\")\n        return \"web_search\"\n    else:\n        # We have relevant documents, so generate answer\n        print(\"---DECISION: GENERATE---\")\n        return \"genLine truncated
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3ab01f36-5628-49ab-bfd3-84bb6f1a1b0f",
+   "metadata": {
+    "id": "3ab01f36-5628-49ab-bfd3-84bb6f1a1b0f"
+   },
+   "source": [
+    "## Build Graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "67854e07-9293-4c3c-bf9a-bc9a605570ee",
+   "metadata": {
+    "id": "67854e07-9293-4c3c-bf9a-bc9a605570ee"
+   },
+   "outputs": [],
+   "source": [
+    "import pprint\n",
+    "\n",
+    "from langgraph.graph import END, StateGraph, START\n",
+    "\n",
+    "workflow = StateGraph(GraphState)\n",
+    "\n",
+    "# Define the nodes\n",
+    "workflow.add_node(\"web_search\", web_search)  # web search\n",
+    "workflow.add_node(\"retrieve\", retrieve)  # retrieve\n",
+    "workflow.add_node(\"grade_documents\", grade_documents)  # grade documents\n",
+    "workflow.add_node(\"generate\", generate)  # rag\n",
+    "workflow.add_node(\"llm_fallback\", llm_fallback)  # llm\n",
+    "\n",
+    "# Build graph\n",
+    "workflow.add_conditional_edges(\n",
+    "    START,\n",
+    "    route_question,\n",
+    "    {\n",
+    "        \"web_search\": \"web_search\",\n",
+    "        \"vectorstore\": \"retrieve\",\n",
+    "        \"llm_fallback\": \"llm_fallback\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"web_search\", \"generate\")\n",
+    "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"grade_documents\",\n",
+    "    decide_to_generate,\n",
+    "    {\n",
+    "        \"web_search\": \"web_search\",\n",
+    "        \"generate\": \"generate\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"generate\",\n",
+    "    grade_generation_v_documents_and_question,\n",
+    "    {\n",
+    "        \"not supported\": \"generate\",  # Hallucinations: re-generate\n",
+    "        \"not useful\": \"web_search\",  # Fails to answer question: fall-back to web-search\n",
+    "        \"useful\": END,\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"llm_fallback\", END)\n",
+    "\n",
+    "# Compile\n",
+    "app = workflow.compile()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "29acc541-d726-4b75-84d1-a215845fe88a",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "29acc541-d726-4b75-84d1-a215845fe88a",
+    "outputId": "47caec8e-54e3-4f89-dfbb-94fc034666f7"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---ROUTE QUESTION---\n",
+      "---ROUTE QUESTION TO WEB SEARCH---\n",
+      "---WEB SEARCH---\n",
+      "\"Node 'web_search':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "'The Bears are expected to draft Caleb Williams with their first pick.'\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run\ninputs = {\n    \"question\": \"What player are the Bears expected to draft first in the 2024 NFL draft?\"\n}\nfor output in app.stream(inputs):\n    for key, value in output.items():\n        # Node\n        pprint.pprint(f\"Node '{key}':\")\n        # Optional: print full state at each node\n    pprint.pprint(\"\\n---\\n\")\n\n# Final generation\npprint.pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11fddd00-58bf-4910-bf36-be9e5bfba778",
+   "metadata": {
+    "id": "11fddd00-58bf-4910-bf36-be9e5bfba778"
+   },
+   "source": [
+    "Trace:\n",
+    "\n",
+    "https://smith.langchain.com/public/623da7bb-84a7-4e53-a63e-7ccd77fb9be5/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "69a985dd-03c6-45af-a67b-b15746a2cb5f",
+   "metadata": {
+    "colab": {
+     "base_uri": "https://localhost:8080/"
+    },
+    "id": "69a985dd-03c6-45af-a67b-b15746a2cb5f",
+    "outputId": "e5f799cc-6f36-494f-c8b2-192de1edb7fc"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---ROUTE QUESTION---\n",
+      "---ROUTE QUESTION TO RAG---\n",
+      "---RETRIEVE---\n",
+      "\"Node 'retrieve':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---ASSESS GRADED DOCUMENTS---\n",
+      "---DECISION: GENERATE---\n",
+      "\"Node 'grade_documents':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "'Sensory, short-term, and long-term memory.'\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run\ninputs = {\"question\": \"What are the types of agent memory?\"}\nfor output in app.stream(inputs):\n    for key, value in output.items():\n        # Node\n        pprint.pprint(f\"Node '{key}':\")\n        # Optional: print full state at each node\n        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n    pprint.pprint(\"\\n---\\n\")\n\n# Final generation\npprint.pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ebf41097-fc4c-4072-95b3-e7e07731ada1",
+   "metadata": {
+    "id": "ebf41097-fc4c-4072-95b3-e7e07731ada1"
+   },
+   "source": [
+    "Trace:\n",
+    "\n",
+    "https://smith.langchain.com/public/57f3973b-6879-4fbe-ae31-9ae524c3a697/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "qPwP_2PNiOjQ",
+   "metadata": {
+    "id": "qPwP_2PNiOjQ"
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---ROUTE QUESTION---\n",
+      "---ROUTE QUESTION TO LLM---\n",
+      "---LLM Fallback---\n",
+      "\"Node 'llm_fallback':\"\n",
+      "'\\n---\\n'\n",
+      "(\"I don't have feelings as an AI assistant, but I'm here to help you with your \"\n",
+      " 'queries. How can I assist you today?')\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Run\ninputs = {\"question\": \"Hello, how are you today?\"}\nfor output in app.stream(inputs):\n    for key, value in output.items():\n        # Node\n        pprint.pprint(f\"Node '{key}':\")\n        # Optional: print full state at each node\n        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n    pprint.pprint(\"\\n---\\n\")\n\n# Final generation\npprint.pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4107c8a4-6171-4c1b-840a-77a3d09f84fc",
+   "metadata": {},
+   "source": [
+    "Trace: \n",
+    "\n",
+    "https://smith.langchain.com/public/1f628ee4-8d2d-451e-aeb1-5d5e0ede2b4f/r"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ce3cda0a-c4bd-41ea-830b-d992f27fde15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    ""
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "provenance": []
+  },
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_adaptive_rag_local.ipynb

@@ -0,0 +1,838 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "3755396d-c4a8-45bd-87d4-00cb56339fe5.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABB0AAAJUCAYAAABZgl4AAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAQdoAMABAAAAAEAAAJUAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdF61mbUAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjU5NjwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xMDUzPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "bb89d3f0-7ade-43a8-a527-4bec45971cf6",
+   "metadata": {},
+   "source": [
+    "# Adaptive RAG using local LLMs\n",
+    "\n",
+    "Adaptive RAG is a strategy for RAG that unites (1) [query analysis](https://blog.langchain.dev/query-construction/) with (2) [active / self-corrective RAG](https://blog.langchain.dev/agentic-rag-with-langgraph/).\n",
+    "\n",
+    "In the [paper](https://arxiv.org/abs/2403.14403), they report query analysis to route across:\n",
+    "\n",
+    "* No Retrieval\n",
+    "* Single-shot RAG\n",
+    "* Iterative RAG\n",
+    "\n",
+    "Let's build on this using LangGraph. \n",
+    "\n",
+    "In our implementation, we will route between:\n",
+    "\n",
+    "* Web search: for questions related to recent events\n",
+    "* Self-corrective RAG: for questions related to our index\n",
+    "\n",
+    "![Screenshot 2024-04-01 at 1.29.15 PM.png](attachment:3755396d-c4a8-45bd-87d4-00cb56339fe5.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8cece98f-a3ed-417e-8b6a-1754e8f9c42a",
+   "metadata": {},
+   "source": [
+    "## Setup\n",
+    "\n",
+    "First, let's install our required packages and set our API keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "88debf5c-6972-415c-b8fb-f65eab203b7a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%capture --no-stderr\n",
+    "%pip install -U langchain-nomic langchain_community tiktoken langchainhub chromadb langchain langgraph tavily-python nomic[local]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2369652a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "\n",
+    "def _set_env(var: str):\n",
+    "    if not os.environ.get(var):\n",
+    "        os.environ[var] = getpass.getpass(f\"{var}: \")\n",
+    "\n",
+    "\n",
+    "_set_env(\"TAVILY_API_KEY\")\n",
+    "_set_env(\"NOMIC_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aea269f6",
+   "metadata": {},
+   "source": [
+    "<div class=\"admonition tip\">\n",
+    "    <p class=\"admonition-title\">Set up <a href=\"https://smith.langchain.com\">LangSmith</a> for LangGraph development</p>\n",
+    "    <p style=\"padding-top: 5px;\">\n",
+    "        Sign up for LangSmith to quickly spot issues and improve the performance of your LangGraph projects. LangSmith lets you use trace data to debug, test, and monitor your LLM apps built with LangGraph — read more about how to get started <a href=\"https://docs.smith.langchain.com\">here</a>. \n",
+    "    </p>\n",
+    "</div>    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6a5d4a26-249b-4551-aa13-6c373429618e",
+   "metadata": {},
+   "source": [
+    "### LLMs\n",
+    "\n",
+    "#### Local Embeddings\n",
+    "\n",
+    "You can use `GPT4AllEmbeddings()` from Nomic, which can access use Nomic's recently released [v1](https://blog.nomic.ai/posts/nomic-embed-text-v1) and [v1.5](https://blog.nomic.ai/posts/nomic-embed-matryoshka) embeddings.\n",
+    "\n",
+    "Follow the documentation [here](https://docs.gpt4all.io/gpt4all_python_embedding.html#supported-embedding-models).\n",
+    "\n",
+    "#### Local LLM\n",
+    "\n",
+    "(1) Download [Ollama app](https://ollama.ai/).\n",
+    "\n",
+    "(2) Download a `Mistral` model from various Mistral versions [here](https://ollama.ai/library/mistral) and Mixtral versions [here](https://ollama.ai/library/mixtral) available. Also, try one of the [quantized command-R models](https://ollama.com/library/command-r).\n",
+    "\n",
+    "```\n",
+    "ollama pull mistral\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "af8379bd-7eae-4ba6-b632-12e89eab9920",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Ollama model name\n",
+    "local_llm = \"mistral\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "04718a0c-7a48-4243-97a2-940a0239cc12",
+   "metadata": {},
+   "source": [
+    "## Create Index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "f9ff6b99-080d-4827-b2cb-f775543d76f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_community.document_loaders import WebBaseLoader\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_nomic.embeddings import NomicEmbeddings\n",
+    "\n",
+    "urls = [\n",
+    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+    "]\n",
+    "\n",
+    "docs = [WebBaseLoader(url).load() for url in urls]\n",
+    "docs_list = [item for sublist in docs for item in sublist]\n",
+    "\n",
+    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+    "    chunk_size=250, chunk_overlap=0\n",
+    ")\n",
+    "doc_splits = text_splitter.split_documents(docs_list)\n",
+    "\n",
+    "# Add to vectorDB\n",
+    "vectorstore = Chroma.from_documents(\n",
+    "    documents=doc_splits,\n",
+    "    collection_name=\"rag-chroma\",\n",
+    "    embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n",
+    ")\n",
+    "retriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2f3eb922-27a1-4a72-a727-85fbf5b3daf1",
+   "metadata": {},
+   "source": [
+    "## LLMs\n",
+    "\n",
+    "Note: tested cmd-R on Mac M2 32GB and [latency is ~52 sec for RAG generation](https://smith.langchain.com/public/3998fe48-efc2-4d18-9069-972643d0982d/r)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "7045e064-e666-4aea-9111-6e9d2007f27e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'datasource': 'vectorstore'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Router\n",
+    "\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from langchain_community.chat_models import ChatOllama\n",
+    "from langchain_core.output_parsers import JsonOutputParser\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+    "\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are an expert at routing a user question to a vectorstore or web search. \\n\n",
+    "    Use the vectorstore for questions on LLM  agents, prompt engineering, and adversarial attacks. \\n\n",
+    "    You do not need to be stringent with the keywords in the question related to these topics. \\n\n",
+    "    Otherwise, use web-search. Give a binary choice 'web_search' or 'vectorstore' based on the question. \\n\n",
+    "    Return the a JSON with a single key 'datasource' and no premable or explanation. \\n\n",
+    "    Question to route: {question}\"\"\",\n",
+    "    input_variables=[\"question\"],\n",
+    ")\n",
+    "\n",
+    "question_router = prompt | llm | JsonOutputParser()\n",
+    "question = \"llm agent memory\"\n",
+    "docs = retriever.get_relevant_documents(question)\n",
+    "doc_txt = docs[1].page_content\n",
+    "print(question_router.invoke({\"question\": question}))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "813cdcef-8b75-4214-a2ed-b89077b3d287",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'score': 'yes'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Retrieval Grader\n",
+    "\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from langchain_community.chat_models import ChatOllama\n",
+    "from langchain_core.output_parsers import JsonOutputParser\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+    "\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+    "    Here is the retrieved document: \\n\\n {document} \\n\\n\n",
+    "    Here is the user question: {question} \\n\n",
+    "    If the document contains keywords related to the user question, grade it as relevant. \\n\n",
+    "    It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
+    "    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n",
+    "    Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\"\"\",\n",
+    "    input_variables=[\"question\", \"document\"],\n",
+    ")\n",
+    "\n",
+    "retrieval_grader = prompt | llm | JsonOutputParser()\n",
+    "question = \"agent memory\"\n",
+    "docs = retriever.get_relevant_documents(question)\n",
+    "doc_txt = docs[1].page_content\n",
+    "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "aeb8b373-0289-4dec-bd4b-8b2701200301",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " In an LLM-powered autonomous agent system, the Large Language Model (LLM) functions as the agent's brain. The agent has key components including memory, planning, and reflection mechanisms. The memory component is a long-term memory module that records a comprehensive list of agents’ experience in natural language. It includes a memory stream, which is an external database for storing past experiences. The reflection mechanism synthesizes memories into higher-level inferences over time and guides the agent's future behavior.\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Generate\n",
+    "\n",
+    "from langchain import hub\n",
+    "from langchain_community.chat_models import ChatOllama\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, temperature=0)\n",
+    "\n",
+    "\n",
+    "# Post-processing\n",
+    "def format_docs(docs):\n",
+    "    return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+    "\n",
+    "\n",
+    "# Chain\n",
+    "rag_chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "# Run\n",
+    "question = \"agent memory\"\n",
+    "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+    "print(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "38345cff-e2d0-436e-aa09-599522a61eed",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'score': 'yes'}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Hallucination Grader\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are a grader assessing whether an answer is grounded in / supported by a set of facts. \\n \n",
+    "    Here are the facts:\n",
+    "    \\n ------- \\n\n",
+    "    {documents} \n",
+    "    \\n ------- \\n\n",
+    "    Here is the answer: {generation}\n",
+    "    Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \\n\n",
+    "    Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
+    "    input_variables=[\"generation\", \"documents\"],\n",
+    ")\n",
+    "\n",
+    "hallucination_grader = prompt | llm | JsonOutputParser()\n",
+    "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "9771caa1-5542-47c3-8354-aeeafcf51964",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'score': 'yes'}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Answer Grader\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are a grader assessing whether an answer is useful to resolve a question. \\n \n",
+    "    Here is the answer:\n",
+    "    \\n ------- \\n\n",
+    "    {generation} \n",
+    "    \\n ------- \\n\n",
+    "    Here is the question: {question}\n",
+    "    Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \\n\n",
+    "    Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
+    "    input_variables=[\"generation\", \"question\"],\n",
+    ")\n",
+    "\n",
+    "answer_grader = prompt | llm | JsonOutputParser()\n",
+    "answer_grader.invoke({\"question\": question, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "830ba5f7-9c8d-4c01-83b1-e4d51d40d48f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "' What is agent memory and how can it be effectively utilized in vector database retrieval?'"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Question Re-writer\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "re_write_prompt = PromptTemplate(\n",
+    "    template=\"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+    "     for vectorstore retrieval. Look at the initial and formulate an improved question. \\n\n",
+    "     Here is the initial question: \\n\\n {question}. Improved question with no preamble: \\n \"\"\",\n",
+    "    input_variables=[\"generation\", \"question\"],\n",
+    ")\n",
+    "\n",
+    "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+    "question_rewriter.invoke({\"question\": question})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "686c9bb1-5069-45f9-8a7e-cba34fe07dd9",
+   "metadata": {},
+   "source": [
+    "## Web Search Tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "6c3c1c70-ff84-41e8-bf72-738ed52f2dde",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Search\n",
+    "\n",
+    "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+    "\n",
+    "web_search_tool = TavilySearchResults(k=3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "630d1751-a20b-4858-b3fd-0312de4f3ad7",
+   "metadata": {},
+   "source": [
+    "# Graph \n",
+    "\n",
+    "Capture the flow in as a graph.\n",
+    "\n",
+    "## Graph state"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "6e09087e-b2a9-437a-abee-129e426df799",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List\n",
+    "\n",
+    "from typing_extensions import TypedDict\n",
+    "\n",
+    "\n",
+    "class GraphState(TypedDict):\n",
+    "    \"\"\"\n",
+    "    Represents the state of our graph.\n",
+    "\n",
+    "    Attributes:\n",
+    "        question: question\n",
+    "        generation: LLM generation\n",
+    "        documents: list of documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question: str\n",
+    "    generation: str\n",
+    "    documents: List[str]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "7c5fa507-77ae-426a-a65f-f518b9525bd0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Nodes\n",
+    "\n",
+    "from langchain.schema import Document\n",
+    "\n",
+    "\n",
+    "def retrieve(state):\n",
+    "    \"\"\"\n",
+    "    Retrieve documents\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, documents, that contains retrieved documents\n",
+    "    \"\"\"\n",
+    "    print(\"---RETRIEVE---\")\n",
+    "    question = state[\"question\"]\n",
+    "\n",
+    "    # Retrieval\n",
+    "    documents = retriever.get_relevant_documents(question)\n",
+    "    return {\"documents\": documents, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def generate(state):\n",
+    "    \"\"\"\n",
+    "    Generate answer\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, generation, that contains LLM generation\n",
+    "    \"\"\"\n",
+    "    print(\"---GENERATE---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # RAG generation\n",
+    "    generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+    "\n",
+    "\n",
+    "def grade_documents(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the retrieved documents are relevant to the question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with only filtered relevant documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Score each doc\n",
+    "    filtered_docs = []\n",
+    "    for d in documents:\n",
+    "        score = retrieval_grader.invoke(\n",
+    "            {\"question\": question, \"document\": d.page_content}\n",
+    "        )\n",
+    "        grade = score[\"score\"]\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+    "            filtered_docs.append(d)\n",
+    "        else:\n",
+    "            print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+    "            continue\n",
+    "    return {\"documents\": filtered_docs, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def transform_query(state):\n",
+    "    \"\"\"\n",
+    "    Transform the query to produce a better question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates question key with a re-phrased question\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---TRANSFORM QUERY---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Re-write question\n",
+    "    better_question = question_rewriter.invoke({\"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": better_question}\n",
+    "\n",
+    "\n",
+    "def web_search(state):\n",
+    "    \"\"\"\n",
+    "    Web search based on the re-phrased question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with appended web results\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---WEB SEARCH---\")\n",
+    "    question = state[\"question\"]\n",
+    "\n",
+    "    # Web search\n",
+    "    docs = web_search_tool.invoke({\"query\": question})\n",
+    "    web_results = \"\\n\".join([d[\"content\"] for d in docs])\n",
+    "    web_results = Document(page_content=web_results)\n",
+    "\n",
+    "    return {\"documents\": web_results, \"question\": question}\n",
+    "\n",
+    "\n",
+    "### Edges ###\n",
+    "\n",
+    "\n",
+    "def route_question(state):\n",
+    "    \"\"\"\n",
+    "    Route question to web search or RAG.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---ROUTE QUESTION---\")\n",
+    "    question = state[\"question\"]\n",
+    "    print(question)\n",
+    "    source = question_router.invoke({\"question\": question})\n",
+    "    print(source)\n",
+    "    print(source[\"datasource\"])\n",
+    "    if source[\"datasource\"] == \"web_search\":\n",
+    "        print(\"---ROUTE QUESTION TO WEB SEARCH---\")\n",
+    "        return \"web_search\"\n",
+    "    elif source[\"datasource\"] == \"vectorstore\":\n",
+    "        print(\"---ROUTE QUESTION TO RAG---\")\n",
+    "        return \"vectorstore\"\n",
+    "\n",
+    "\n",
+    "def decide_to_generate(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether to generate an answer, or re-generate a question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Binary decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+    "    state[\"question\"]\n",
+    "    filtered_documents = state[\"documents\"]\n",
+    "\n",
+    "    if not filtered_documents:\n",
+    "        # All documents have been filtered check_relevance\n",
+    "        # We will re-generate a new query\n",
+    "        print(\n",
+    "            \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+    "        )\n",
+    "        return \"transform_query\"\n",
+    "    else:\n",
+    "        # We have relevant documents, so generate answer\n",
+    "        print(\"---DECISION: GENERATE---\")\n",
+    "        return \"generate\"\n",
+    "\n",
+    "\n",
+    "def grade_generation_v_documents_and_question(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the generation is grounded in the document and answers question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK HALLUCINATIONS---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "    generation = state[\"generation\"]\n",
+    "\n",
+    "    score = hallucination_grader.invoke(\n",
+    "        {\"documents\": documents, \"generation\": generation}\n",
+    "    )\n",
+    "    grade = score[\"score\"]\n",
+    "\n",
+    "    # Check hallucination\n",
+    "    if grade == \"yes\":\n",
+    "        print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
+    "        # Check question-answering\n",
+    "        print(\"---GRADE GENERATION vs QUESTION---\")\n",
+    "        score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
+    "        grade = score[\"score\"]\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
+    "            return \"useful\"\n",
+    "        else:\n",
+    "            print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
+    "            return \"not useful\"\n",
+    "    else:\n",
+    "        pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
+    "        return \"not supported\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ed7d8eb6-31d7-4ab5-8a88-7081b64582bb",
+   "metadata": {},
+   "source": [
+    "## Build Graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "450eb313-ca75-4a43-b57e-7034bd3f40bf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langgraph.graph import END, StateGraph, START\n",
+    "\n",
+    "workflow = StateGraph(GraphState)\n",
+    "\n",
+    "# Define the nodes\n",
+    "workflow.add_node(\"web_search\", web_search)  # web search\n",
+    "workflow.add_node(\"retrieve\", retrieve)  # retrieve\n",
+    "workflow.add_node(\"grade_documents\", grade_documents)  # grade documents\n",
+    "workflow.add_node(\"generate\", generate)  # generate\n",
+    "workflow.add_node(\"transform_query\", transform_query)  # transform_query\n",
+    "\n",
+    "# Build graph\n",
+    "workflow.add_conditional_edges(\n",
+    "    START,\n",
+    "    route_question,\n",
+    "    {\n",
+    "        \"web_search\": \"web_search\",\n",
+    "        \"vectorstore\": \"retrieve\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"web_search\", \"generate\")\n",
+    "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"grade_documents\",\n",
+    "    decide_to_generate,\n",
+    "    {\n",
+    "        \"transform_query\": \"transform_query\",\n",
+    "        \"generate\": \"generate\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"transform_query\", \"retrieve\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"generate\",\n",
+    "    grade_generation_v_documents_and_question,\n",
+    "    {\n",
+    "        \"not supported\": \"generate\",\n",
+    "        \"useful\": END,\n",
+    "        \"not useful\": \"transform_query\",\n",
+    "    },\n",
+    ")\n",
+    "\n",
+    "# Compile\n",
+    "app = workflow.compile()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "b095c1db-8bd1-4a34-937c-1a9b74ae74ff",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---ROUTE QUESTION---\n",
+      "What is the AlphaCodium paper about?\n",
+      "{'datasource': 'web_search'}\n",
+      "web_search\n",
+      "---ROUTE QUESTION TO WEB SEARCH---\n",
+      "---WEB SEARCH---\n",
+      "\"Node 'web_search':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "(' The AlphaCodium paper introduces a new approach for code generation by '\n",
+      " 'Large Language Models (LLMs). It presents AlphaCodium, an iterative process '\n",
+      " 'that involves generating additional data to aid the flow, and testing it on '\n",
+      " 'the CodeContests dataset. The results show that AlphaCodium outperforms '\n",
+      " \"DeepMind's AlphaCode and AlphaCode2 without fine-tuning a model. The \"\n",
+      " 'approach includes a pre-processing phase for problem reasoning in natural '\n",
+      " 'language and an iterative code generation phase with runs and fixes against '\n",
+      " 'tests.')\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Run\n",
+    "inputs = {\"question\": \"What is the AlphaCodium paper about?\"}\n",
+    "for output in app.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        # Node\n",
+    "        pprint(f\"Node '{key}':\")\n",
+    "        # Optional: print full state at each node\n",
+    "        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+    "    pprint(\"\\n---\\n\")\n",
+    "\n",
+    "# Final generation\n",
+    "pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "644c7293-9cb5-4236-ba08-1e63b0309cb7",
+   "metadata": {},
+   "source": [
+    "Trace: \n",
+    "\n",
+    "https://smith.langchain.com/public/81813813-be53-403c-9877-afcd5786ca2e/r"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_agentic_rag.ipynb

@@ -0,0 +1,539 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "425fb020-e864-40ce-a31f-8da40c73d14b",
+   "metadata": {},
+   "source": [
+    "# Agentic RAG\n",
+    "\n",
+    "[Retrieval Agents](https://python.langchain.com/v0.2/docs/tutorials/qa_chat_history/#agents) are useful when we want to make decisions about whether to retrieve from an index.\n",
+    "\n",
+    "To implement a retrieval agent, we simple need to give an LLM access to a retriever tool.\n",
+    "\n",
+    "We can incorporate this into [LangGraph](https://langchain-ai.github.io/langgraph/).\n",
+    "\n",
+    "## Setup\n",
+    "\n",
+    "First, let's download the required packages and set our API keys:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "969fb438",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture --no-stderr\n",
+    "%pip install -U --quiet langchain-community tiktoken langchain-openai langchainhub chromadb langchain langgraph langchain-text-splitters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e4958a8c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "\n",
+    "def _set_env(key: str):\n",
+    "    if key not in os.environ:\n",
+    "        os.environ[key] = getpass.getpass(f\"{key}:\")\n",
+    "\n",
+    "\n",
+    "_set_env(\"OPENAI_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3d07e8d4",
+   "metadata": {},
+   "source": [
+    "<div class=\"admonition tip\">\n",
+    "    <p class=\"admonition-title\">Set up <a href=\"https://smith.langchain.com\">LangSmith</a> for LangGraph development</p>\n",
+    "    <p style=\"padding-top: 5px;\">\n",
+    "        Sign up for LangSmith to quickly spot issues and improve the performance of your LangGraph projects. LangSmith lets you use trace data to debug, test, and monitor your LLM apps built with LangGraph — read more about how to get started <a href=\"https://docs.smith.langchain.com\">here</a>. \n",
+    "    </p>\n",
+    "</div>    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c74e4532",
+   "metadata": {},
+   "source": [
+    "## Retriever\n",
+    "\n",
+    "First, we index 3 blog posts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "e50c9efe-4abe-42fa-b35a-05eeeede9ec6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import WebBaseLoader\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "\n",
+    "urls = [\n",
+    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+    "]\n",
+    "\n",
+    "docs = [WebBaseLoader(url).load() for url in urls]\n",
+    "docs_list = [item for sublist in docs for item in sublist]\n",
+    "\n",
+    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+    "    chunk_size=100, chunk_overlap=50\n",
+    ")\n",
+    "doc_splits = text_splitter.split_documents(docs_list)\n",
+    "\n",
+    "# Add to vectorDB\n",
+    "vectorstore = Chroma.from_documents(\n",
+    "    documents=doc_splits,\n",
+    "    collection_name=\"rag-chroma\",\n",
+    "    embedding=OpenAIEmbeddings(),\n",
+    ")\n",
+    "retriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "225d2277-45b2-4ae8-a7d6-62b07fb4a002",
+   "metadata": {},
+   "source": [
+    "Then we create a retriever tool."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "0b97bdd8-d7e3-444d-ac96-5ef4725f9048",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.tools.retriever import create_retriever_tool\n",
+    "\n",
+    "retriever_tool = create_retriever_tool(\n",
+    "    retriever,\n",
+    "    \"retrieve_blog_posts\",\n",
+    "    \"Search and return information about Lilian Weng blog posts on LLM agents, prompt engineering, and adversarial attacks on LLMs.\",\n",
+    ")\n",
+    "\n",
+    "tools = [retriever_tool]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fe6e8f78-1ef7-42ad-b2bf-835ed5850553",
+   "metadata": {},
+   "source": [
+    "## Agent State\n",
+    " \n",
+    "We will define a graph.\n",
+    "\n",
+    "A `state` object that it passes around to each node.\n",
+    "\n",
+    "Our state will be a list of `messages`.\n",
+    "\n",
+    "Each node in our graph will append to it."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "0e378706-47d5-425a-8ba0-57b9acffbd0c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import Annotated, Sequence, TypedDict\n",
+    "\n",
+    "from langchain_core.messages import BaseMessage\n",
+    "\n",
+    "from langgraph.graph.message import add_messages\n",
+    "\n",
+    "\n",
+    "class AgentState(TypedDict):\n",
+    "    # The add_messages function defines how an update should be processed\n",
+    "    # Default is to replace. add_messages says \"append\"\n",
+    "    messages: Annotated[Sequence[BaseMessage], add_messages]"
+   ]
+  },
+  {
+   "attachments": {
+    "7ad1a116-28d7-473f-8cff-5f2efd0bf118.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABk8AAAJNCAYAAACY1uLnAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAZPoAMABAAAAAEAAAJNAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdNqyY5cAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjU4OTwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNjE1PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "dc949d42-8a34-4231-bff0-b8198975e2ce",
+   "metadata": {},
+   "source": [
+    "## Nodes and Edges\n",
+    "\n",
+    "We can lay out an agentic RAG graph like this:\n",
+    "\n",
+    "* The state is a set of messages\n",
+    "* Each node will update (append to) state\n",
+    "* Conditional edges decide which node to visit next\n",
+    "\n",
+    "![Screenshot 2024-02-14 at 3.43.58 PM.png](attachment:7ad1a116-28d7-473f-8cff-5f2efd0bf118.png)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "278d1d83-dda6-4de4-bf8b-be9965c227fa",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "********************Prompt[rlm/rag-prompt]********************\n",
+      "================================\u001B[1m Human Message \u001B[0m=================================\n",
+      "\n",
+      "You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. Use three sentences maximum and keep the answer concise.\n",
+      "Question: \u001B[33;1m\u001B[1;3m{question}\u001B[0m \n",
+      "Context: \u001B[33;1m\u001B[1;3m{context}\u001B[0m \n",
+      "Answer:\n"
+     ]
+    }
+   ],
+   "source": [
+    "from typing import Annotated, Literal, Sequence, TypedDict\n",
+    "\n",
+    "from langchain import hub\n",
+    "from langchain_core.messages import BaseMessage, HumanMessage\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "from langchain_core.prompts import PromptTemplate\n",
+    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "from langgraph.prebuilt import tools_condition\n",
+    "\n",
+    "### Edges\n",
+    "\n",
+    "\n",
+    "def grade_documents(state) -> Literal[\"generate\", \"rewrite\"]:\n",
+    "    \"\"\"\n",
+    "    Determines whether the retrieved documents are relevant to the question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (messages): The current state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: A decision for whether the documents are relevant or not\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK RELEVANCE---\")\n",
+    "\n",
+    "    # Data model\n",
+    "    class grade(BaseModel):\n",
+    "        \"\"\"Binary score for relevance check.\"\"\"\n",
+    "\n",
+    "        binary_score: str = Field(description=\"Relevance score 'yes' or 'no'\")\n",
+    "\n",
+    "    # LLM\n",
+    "    model = ChatOpenAI(temperature=0, model=\"gpt-4o\", streaming=True)\n",
+    "\n",
+    "    # LLM with tool and validation\n",
+    "    llm_with_tool = model.with_structured_output(grade)\n",
+    "\n",
+    "    # Prompt\n",
+    "    prompt = PromptTemplate(\n",
+    "        template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+    "        Here is the retrieved document: \\n\\n {context} \\n\\n\n",
+    "        Here is the user question: {question} \\n\n",
+    "        If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
+    "        Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\",\n",
+    "        input_variables=[\"context\", \"question\"],\n",
+    "    )\n",
+    "\n",
+    "    # Chain\n",
+    "    chain = prompt | llm_with_tool\n",
+    "\n",
+    "    messages = state[\"messages\"]\n",
+    "    last_message = messages[-1]\n",
+    "\n",
+    "    question = messages[0].content\n",
+    "    docs = last_message.content\n",
+    "\n",
+    "    scored_result = chain.invoke({\"question\": question, \"context\": docs})\n",
+    "\n",
+    "    score = scored_result.binary_score\n",
+    "\n",
+    "    if score == \"yes\":\n",
+    "        print(\"---DECISION: DOCS RELEVANT---\")\n",
+    "        return \"generate\"\n",
+    "\n",
+    "    else:\n",
+    "        print(\"---DECISION: DOCS NOT RELEVANT---\")\n",
+    "        print(score)\n",
+    "        return \"rewrite\"\n",
+    "\n",
+    "\n",
+    "### Nodes\n",
+    "\n",
+    "\n",
+    "def agent(state):\n",
+    "    \"\"\"\n",
+    "    Invokes the agent model to generate a response based on the current state. Given\n",
+    "    the question, it will decide to retrieve using the retriever tool, or simply end.\n",
+    "\n",
+    "    Args:\n",
+    "        state (messages): The current state\n",
+    "\n",
+    "    Returns:\n",
+    "        dict: The updated state with the agent response appended to messages\n",
+    "    \"\"\"\n",
+    "    print(\"---CALL AGENT---\")\n",
+    "    messages = state[\"messages\"]\n",
+    "    model = ChatOpenAI(temperature=0, streaming=True, model=\"gpt-4-turbo\")\n",
+    "    model = model.bind_tools(tools)\n",
+    "    response = model.invoke(messages)\n",
+    "    # We return a list, because this will get added to the existing list\n",
+    "    return {\"messages\": [response]}\n",
+    "\n",
+    "\n",
+    "def rewrite(state):\n",
+    "    \"\"\"\n",
+    "    Transform the query to produce a better question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (messages): The current state\n",
+    "\n",
+    "    Returns:\n",
+    "        dict: The updated state with re-phrased question\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---TRANSFORM QUERY---\")\n",
+    "    messages = state[\"messages\"]\n",
+    "    question = messages[0].content\n",
+    "\n",
+    "    msg = [\n",
+    "        HumanMessage(\n",
+    "            content=f\"\"\" \\n \n",
+    "    Look at the input and try to reason about the underlying semantic intent / meaning. \\n \n",
+    "    Here is the initial question:\n",
+    "    \\n ------- \\n\n",
+    "    {question} \n",
+    "    \\n ------- \\n\n",
+    "    Formulate an improved question: \"\"\",\n",
+    "        )\n",
+    "    ]\n",
+    "\n",
+    "    # Grader\n",
+    "    model = ChatOpenAI(temperature=0, model=\"gpt-4-0125-preview\", streaming=True)\n",
+    "    response = model.invoke(msg)\n",
+    "    return {\"messages\": [response]}\n",
+    "\n",
+    "\n",
+    "def generate(state):\n",
+    "    \"\"\"\n",
+    "    Generate answer\n",
+    "\n",
+    "    Args:\n",
+    "        state (messages): The current state\n",
+    "\n",
+    "    Returns:\n",
+    "         dict: The updated state with re-phrased question\n",
+    "    \"\"\"\n",
+    "    print(\"---GENERATE---\")\n",
+    "    messages = state[\"messages\"]\n",
+    "    question = messages[0].content\n",
+    "    last_message = messages[-1]\n",
+    "\n",
+    "    docs = last_message.content\n",
+    "\n",
+    "    # Prompt\n",
+    "    prompt = hub.pull(\"rlm/rag-prompt\")\n",
+    "\n",
+    "    # LLM\n",
+    "    llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0, streaming=True)\n",
+    "\n",
+    "    # Post-processing\n",
+    "    def format_docs(docs):\n",
+    "        return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+    "\n",
+    "    # Chain\n",
+    "    rag_chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "    # Run\n",
+    "    response = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+    "    return {\"messages\": [response]}\n",
+    "\n",
+    "\n",
+    "print(\"*\" * 20 + \"Prompt[rlm/rag-prompt]\" + \"*\" * 20)\n",
+    "prompt = hub.pull(\"rlm/rag-prompt\").pretty_print()  # Show what the prompt looks like"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "955882ef-7467-48db-ae51-de441f2fc3a7",
+   "metadata": {},
+   "source": [
+    "## Graph\n",
+    "\n",
+    "* Start with an agent, `call_model`\n",
+    "* Agent make a decision to call a function\n",
+    "* If so, then `action` to call tool (retriever)\n",
+    "* Then call agent with the tool output added to messages (`state`)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "8718a37f-83c2-4f16-9850-e61e0f49c3d4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langgraph.graph import END, StateGraph, START\n",
+    "from langgraph.prebuilt import ToolNode\n",
+    "\n",
+    "# Define a new graph\n",
+    "workflow = StateGraph(AgentState)\n",
+    "\n",
+    "# Define the nodes we will cycle between\n",
+    "workflow.add_node(\"agent\", agent)  # agent\n",
+    "retrieve = ToolNode([retriever_tool])\n",
+    "workflow.add_node(\"retrieve\", retrieve)  # retrieval\n",
+    "workflow.add_node(\"rewrite\", rewrite)  # Re-writing the question\n",
+    "workflow.add_node(\n",
+    "    \"generate\", generate\n",
+    ")  # Generating a response after we know the documents are relevant\n",
+    "# Call agent node to decide to retrieve or not\n",
+    "workflow.add_edge(START, \"agent\")\n",
+    "\n",
+    "# Decide whether to retrieve\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"agent\",\n",
+    "    # Assess agent decision\n",
+    "    tools_condition,\n",
+    "    {\n",
+    "        # Translate the condition outputs to nodes in our graph\n",
+    "        \"tools\": \"retrieve\",\n",
+    "        END: END,\n",
+    "    },\n",
+    ")\n",
+    "\n",
+    "# Edges taken after the `action` node is called.\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"retrieve\",\n",
+    "    # Assess agent decision\n",
+    "    grade_documents,\n",
+    ")\n",
+    "workflow.add_edge(\"generate\", END)\n",
+    "workflow.add_edge(\"rewrite\", \"agent\")\n",
+    "\n",
+    "# Compile\n",
+    "graph = workflow.compile()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "7b5a1d35",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAGVATEDASIAAhEBAxEB/8QAHQABAAIDAQEBAQAAAAAAAAAAAAYHBAUIAwkCAf/EAFgQAAEDBAECAgQHCwYJCgYDAAEAAgMEBQYREgchEzEIFCJBFRYXMlFV0TRWYXF3gZOUlbHSIzZCUnKRCSQzVGJzgqHhNTdTdHWSorKztCVDREeEhcHU8P/EABoBAQACAwEAAAAAAAAAAAAAAAADBAECBQb/xAA9EQEAAQICBQgHBgUFAAAAAAAAAQIRAwQSIVFSkRMUFTFBodHSBVNhgbHB8DI0QmKS4SJjcXKiIzOCssL/2gAMAwEAAhEDEQA/APqmiIgIiICIiAiIgIiICIiDFrbnR2xrXVlXBSNedNM8jWA/i2VifGqyfXFB+tM+1Q7qdSQVmTYtHUQxzx8Ks8JGhw3xj9xWt+L9r+raP9A37FUzOcwsrNNNdMzMxfVbbMfJ0sDJ8tRp6Vlh/GqyfXFB+tM+1PjVZPrig/Wmfaq8+L9r+raP9A37E+L9r+raP9A37FU6Vy+5VxhP0d+buWH8arJ9cUH60z7U+NVk+uKD9aZ9qrz4v2v6to/0DfsT4v2v6to/0DfsTpXL7lXGDo783csP41WT64oP1pn2p8arJ9cUH60z7VXnxftf1bR/oG/Ynxftf1bR/oG/YnSuX3KuMHR35u5Yfxqsn1xQfrTPtT41WT64oP1pn2qvPi/a/q2j/QN+xPi/a/q2j/QN+xOlcvuVcYOjvzdyxBlNlcQBd6Ak9gBUs7/71tFRua2S3QYpdJI6CljkbAS1zYWgg/SDpXkujg41GYwuVoiY1zGv2W8VLMZfkJiL3uIiKVTEREBERAREQEREBERAREQEREBERAREQEREBERAREQEREFfdRf514t/YrP/ACxrwXv1F/nXi39is/8ALGo/kmZ4/hkEM2QX222KGdxZFJcqyOnbI4DZDS8jZ19C876WiZxcOI3f/VT0WSmIwby3Kjuf51bem+KVmQXYTvo6Z0bPCpY/Ellkke2ONjG7G3Oe9oGyB37kBasdcenBYXjqBixYCAXfDVNoE70Pn/gP9y1GWZ1iPUbFbtZLDJj/AFLq5YWmTHKW8UxdPF4jA92+RDeIPIE69oN7gkEcenDnSjSibLk1xadGdbRdQOu96x+3YbVW7Cb9FJd7+22VVDXU0DahsYjc8tZucM5v0OLuRbpkmyDrcrzHq58S6SkqajDcqr45KMV1SbfRRzChZrbmynxQObdHbYy89u2xoqqabpx1ChwS1TPtlRWT2LMYr3asduF2jnq47cyMx+rmqc4sLwZJHN5PIDdDkfdndR8EyjPMsFxuuC/GK2VtkZTUNrrbrCyCy1pdJ4kkzORbISHR6kjD3DgQB71Z0MO8Rqtr7f3QaVdpnXw/ZO7t17slHd7JbLZarzktXerR8N0DbPTxvE9Pto3uSRgadOB9rQ92+RDThYN1evWTdXszxWrxa401utNRBDBXcIAyAOp/EJnInLjzPzODT7Jby4na0HRnpvk2NZNgtZd7T6jDacF+AKp5qIpONUyohIaOLiSHMjLwQNa0Do9luqKkvnT7rDm17qrRHNiN/bSVk19NfBDHbRT03hSeMyRwdr2A7k0EaJ3rS1mnDi9NOvVt7b+DaKq5tVOrX8vFbyKEDrn03cQB1BxYk9gBeqb+NelJ1p6e19VDTU2d4zU1Mz2xxQxXinc+R5Og1oD9kkkAAKtydeyU+nTtbbOv5oXb/UOVyqms6/mhdv8AUOVyr1Poz7r/AMp+FLjekftUiIi6bkCIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiCvuov8AOvFv7FZ/5Y1iSwRzgCSNsgHlyaDpS3KMMosrkopamerppqTn4UlJN4bhyADge3fyC1HyU0P1xe/13/gqGcyUZuqmuK7Wi3VO2Z+brZbN0YOHoVQ0vqFL/m0P/cC/cdLDC7lHDGx3ltrQCtv8lND9cXv9d/4J8lND9cXv9d/4Kh0RPrY4Ss8/wtktai2XyU0P1xe/13/gqi9IulremkPTZ1kvd0Yb7m1ssVZ41Rz3Sz+J4gb27O9kaPuTof8AmxwlnpDC2SstfxzQ9pa4BzSNEHyK2fyU0P1xe/13/gnyU0P1xe/13/gnQ/8ANjhJ0hhbJaX4PpT/APTQ/owv6KGmaQRTxAjuCGBbn5KaH64vf67/AME+Smh+uL3+u/8ABOiJ9bHCWOf4WyUQzr+aF2/1DlcqgtR0gtdZC+Goud4nheNPjfWEtcPoPZTpdfLZeMrg8npXm8z3R4OdmsenHmJp7BERTqIiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAud/TL+5+i35TrH++ZdELnf0y/ufot+U6x/vmQdEIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiIC539Mv7n6LflOsf75l0QuRPSy61dPMjg6Ri055jN0ND1Es9bVCivFPN6vTsMvOaTi88WN2NuOgNjZQddoo9ifUPFc9FQcYyaz5GKYMM5tNfFVeEHFwaXeG48dlj9b8+LvoKkKAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICLU5Dk9BjNOySse90spIhpoGGSaYjzDWjv22Nk6A33ICiNRn2RVLyaSzUNHDvsa2rc+Qj6S1jeI/wC+VLTh1TGl1R7dSajBxMT7MLEXxP8ATH6GnoR1wu9ppKcxWC4f/EbSQPZEEhP8mP7Dw5mvPTWk+a+s3xyy7/N7L/fMqr659HovSF+L3xsobc51kq/WYH0cj2Oladc4JC5rtxv4t2G6d7I04d1tyUb0cU3NMbYx/wDB4dDndJuiUd7uELob9lhjuE7X9jHTNDvVmEf2Xuf9P8ro+S6mVaMy/LI2NYylsjWNGg1vigAfQv0M0y1p2aSyyf6PiTM3+fR/cnJRvRxY5pjbFkooTbupbY5WxX23utHI8RVxyePS/wC0/QLPxvaG/h8lNlHVRVR1q9dFVE2qiwiItGgiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICIiAiIgIiICw7zdqexWqruNU4tp6WJ0r+PmQBvQ+knyA+lZihvVh7m4pGz/5ctxoo5NjY4mpj7fnOh+dS4VMV100z2y3op0qop2ovRMqaqZ9zuenXWqaDKA7k2Ee6Fh/qN7/jO3HuSsxFQPXWsuWRZXW2bGajIm3ez2f4QqpKC/m10VI15k8J7w1jzNIfDd7BHDTe5G1BXVOJVpS9NNsKm0Qv5FybdeodyyWlwupyrJL/AGC3XPB4bnRTY6ZYnVt2PeUHwWkucGmItiPsnmex8koanqLkF1w/BJ/XYa234fSXWupjkk9qqaipfI6N73zsilkfw4gFm2gOed7GgNbI+Wjsh1ksO53mgsrKZ1fWQUbamojpYDPIG+LM86ZG3fm4nyAXPbLRm7sp6XYnluS3Cmkq4b5646y3R7X1UEZp3U7ZZmsjLpGtIBka1jj7WiOTtxbIqWpynELDa7vervViydURY6et+EZY6h9OJyGF8jHAukaHANkPtDWwQSsWJxZ2fWrxdcOaHtLXAOaRogjYIWwwG6vtNzOOyuJonQma3FztmMNOpIf7LQWFv0AuHYMAWmtVuZaLZS0MUtRPHTRNibLVTOmleGjQL5Hkue76XEkn3r+Pe6HKMVkZ/lPhEsHbuWugmDh/ds/mVnAm8zRPVMTxiNX1su1zNEV4U37FsoiKN5wREQEREBERAREQEREBERAREQEREBERAREQEREBERAREQEREBajLbG7JMcrreyQQzSsDoZSezJWkOjcfwBzWn8y26hnUPqzY+mdZjlHdY7hUVl/r22+hp7dRSVL3vJHJzuI01rQeR2d6B0DohbU1TRVFUdcMxNpvCNWyu+EKNkronU849ianeQXQyDs5jte8HYUZyzpFiWcXmG63qztrK6OH1YyCeWNs0PIu8KVjHBsrNknjIHDue3cq08nwk3KqfcrVPHQXVwAl8VhdDUgdgJACCHAdg8dwNbDgAFE6hl+oHllVjVa/R14tDJFPGR9I9pr/wC9oW84WnN8PhfXHHrd7DzOHiLine truncated
+      "text/plain": [
+       "<IPython.core.display.Image object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from IPython.display import Image, display\n",
+    "\n",
+    "try:\n",
+    "    display(Image(graph.get_graph(xray=True).draw_mermaid_png()))\n",
+    "except Exception:\n",
+    "    # This requires some extra dependencies and is optional\n",
+    "    pass"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "7649f05a-cb67-490d-b24a-74d41895139a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---CALL AGENT---\n",
+      "\"Output from node 'agent':\"\n",
+      "'---'\n",
+      "{ 'messages': [ AIMessage(content='', additional_kwargs={'tool_calls': [{'index': 0, 'id': 'call_z36oPZN8l1UC6raxrebqc1bH', 'function': {'arguments': '{\"query\":\"types of agent memory\"}', 'name': 'retrieve_blog_posts'}, 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls'}, id='run-2bad2518-8187-4d8f-8e23-2b9501becb6f-0', tool_calls=[{'name': 'retrieve_blog_posts', 'args': {'query': 'types of agent memory'}, 'id': 'call_z36oPZN8l1UC6raxrebqc1bH'}])]}\n",
+      "'\\n---\\n'\n",
+      "---CHECK RELEVANCE---\n",
+      "---DECISION: DOCS RELEVANT---\n",
+      "\"Output from node 'retrieve':\"\n",
+      "'---'\n",
+      "{ 'messages': [ ToolMessage(content='Table of Contents\\n\\n\\n\\nAgent System Overview\\n\\nComponent One: Planning\\n\\nTask Decomposition\\n\\nSelf-Reflection\\n\\n\\nComponent Two: Memory\\n\\nTypes of Memory\\n\\nMaximum Inner Product Search (MIPS)\\n\\n\\nComponent Three: Tool Use\\n\\nCase Studies\\n\\nScientific Discovery Agent\\n\\nGenerative Agents Simulation\\n\\nProof-of-Concept Examples\\n\\n\\nChallenges\\n\\nCitation\\n\\nReferences\\n\\nPlanning\\n\\nSubgoal and decomposition: The agent breaks down large tasks into smaller, manageable subgoals, enabling efficient handling of complex tasks.\\nReflection and refinement: The agent can do self-criticism and self-reflection over past actions, learn from mistakes and refine them for future steps, thereby improving the quality of final results.\\n\\n\\nMemory\\n\\nMemory\\n\\nShort-term memory: I would consider all the in-context learning (See Prompt Engineering) as utilizing short-term memory of the model to learn.\\nLong-term memory: This provides the agent with the capability to retain and recall (infinite) information over extended periods, often by leveraging an external vector store and fast retrieval.\\n\\n\\nTool use\\n\\nThe design of generative agents combines LLM with memory, planning and reflection mechanisms to enable agents to behave conditioned on past experience, as well as to interact with other agents.', name='retrieve_blog_posts', id='d815f283-868c-4660-a1c6-5f6e5373ca06', tool_call_id='call_z36oPZN8l1UC6raxrebqc1bH')]}\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "\"Output from node 'generate':\"\n",
+      "'---'\n",
+      "{ 'messages': [ 'Lilian Weng discusses short-term and long-term memory in '\n",
+      "                'agent systems. Short-term memory is used for in-context '\n",
+      "                'learning, while long-term memory allows agents to retain and '\n",
+      "                'recall information over extended periods.']}\n",
+      "'\\n---\\n'\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pprint\n",
+    "\n",
+    "inputs = {\n",
+    "    \"messages\": [\n",
+    "        (\"user\", \"What does Lilian Weng say about the types of agent memory?\"),\n",
+    "    ]\n",
+    "}\n",
+    "for output in graph.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        pprint.pprint(f\"Output from node '{key}':\")\n",
+    "        pprint.pprint(\"---\")\n",
+    "        pprint.pprint(value, indent=2, width=80, depth=None)\n",
+    "    pprint.pprint(\"\\n---\\n\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_crag.ipynb

@@ -0,0 +1,701 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "683fae34-980f-43f0-a9c2-9894bebd9157.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABqIAAALXCAYAAADmN1EDAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAaioAMABAAAAAEAAALXAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdNkXFXsAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjcyNzwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNjk4PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "8889a307-fa3f-4d38-9127-d41e4686ae47",
+   "metadata": {},
+   "source": [
+    "# Corrective RAG (CRAG)\n",
+    "\n",
+    "Corrective-RAG (CRAG) is a strategy for RAG that incorporates self-reflection / self-grading on retrieved documents. \n",
+    "\n",
+    "In the paper [here](https://arxiv.org/pdf/2401.15884.pdf), a few steps are taken:\n",
+    "\n",
+    "* If at least one document exceeds the threshold for relevance, then it proceeds to generation\n",
+    "* Before generation, it performs knowledge refinement\n",
+    "* This partitions the document into \"knowledge strips\"\n",
+    "* It grades each strip, and filters our irrelevant ones\n",
+    "* If all documents fall below the relevance threshold or if the grader is unsure, then the framework seeks an additional datasource\n",
+    "* It will use web search to supplement retrieval\n",
+    " \n",
+    "We will implement some of these ideas from scratch using [LangGraph](https://langchain-ai.github.io/langgraph/):\n",
+    "\n",
+    "* Let's skip the knowledge refinement phase as a first pass. This can be added back as a node, if desired. \n",
+    "* If *any* documents are irrelevant, let's opt to supplement retrieval with web search. \n",
+    "* We'll use [Tavily Search](https://python.langchain.com/v0.2/docs/integrations/tools/tavily_search/) for web search.\n",
+    "* Let's use query re-writing to optimize the query for web search.\n",
+    "\n",
+    "![Screenshot 2024-04-01 at 9.28.30 AM.png](attachment:683fae34-980f-43f0-a9c2-9894bebd9157.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4931ac25-99f9-4f04-b3d1-4683f7853667",
+   "metadata": {},
+   "source": [
+    "## Setup\n",
+    "\n",
+    "First, let's download our required packages and set our API keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "568c84d6-9df6-4b7b-b50d-476c0a64a04b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph tavily-python"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "74710419-158d-4270-931c-de83db7b580d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "\n",
+    "def _set_env(key: str):\n",
+    "    if key not in os.environ:\n",
+    "        os.environ[key] = getpass.getpass(f\"{key}:\")\n",
+    "\n",
+    "\n",
+    "_set_env(\"OPENAI_API_KEY\")\n",
+    "_set_env(\"TAVILY_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3adde047",
+   "metadata": {},
+   "source": [
+    "<div class=\"admonition tip\">\n",
+    "    <p class=\"admonition-title\">Set up <a href=\"https://smith.langchain.com\">LangSmith</a> for LangGraph development</p>\n",
+    "    <p style=\"padding-top: 5px;\">\n",
+    "        Sign up for LangSmith to quickly spot issues and improve the performance of your LangGraph projects. LangSmith lets you use trace data to debug, test, and monitor your LLM apps built with LangGraph — read more about how to get started <a href=\"https://docs.smith.langchain.com\">here</a>. \n",
+    "    </p>\n",
+    "</div>    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a21f32d2-92ce-4995-b309-99347bafe3be",
+   "metadata": {},
+   "source": [
+    "## Create Index\n",
+    " \n",
+    "Let's index 3 blog posts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "3a566a30-cf0e-4330-ad4d-9bf994bdfa86",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_community.document_loaders import WebBaseLoader\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "\n",
+    "urls = [\n",
+    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+    "]\n",
+    "\n",
+    "docs = [WebBaseLoader(url).load() for url in urls]\n",
+    "docs_list = [item for sublist in docs for item in sublist]\n",
+    "\n",
+    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+    "    chunk_size=250, chunk_overlap=0\n",
+    ")\n",
+    "doc_splits = text_splitter.split_documents(docs_list)\n",
+    "\n",
+    "# Add to vectorDB\n",
+    "vectorstore = Chroma.from_documents(\n",
+    "    documents=doc_splits,\n",
+    "    collection_name=\"rag-chroma\",\n",
+    "    embedding=OpenAIEmbeddings(),\n",
+    ")\n",
+    "retriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6fca2db8-8d68-42b0-981d-4be5ccdbe293",
+   "metadata": {},
+   "source": [
+    "## LLMs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "7ece414c-2df5-4ffd-aa82-550a65775261",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "binary_score='yes'\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Retrieval Grader\n",
+    "\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeDocuments(BaseModel):\n",
+    "    \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+    "    If the document contains keyword(s) or semantic meaning related to the question, grade it as relevant. \\n\n",
+    "    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
+    "grade_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "retrieval_grader = grade_prompt | structured_llm_grader\n",
+    "question = \"agent memory\"\n",
+    "docs = retriever.get_relevant_documents(question)\n",
+    "doc_txt = docs[1].page_content\n",
+    "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "a207c85f-e414-46b7-8999-4c0ead1493da",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The design of generative agents combines LLM with memory, planning, and reflection mechanisms to enable agents to behave conditioned on past experience. Memory stream is a long-term memory module that records a comprehensive list of agents' experience in natural language. Short-term memory is utilized for in-context learning, while long-term memory allows agents to retain and recall information over extended periods.\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Generate\n",
+    "\n",
+    "from langchain import hub\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
+    "\n",
+    "\n",
+    "# Post-processing\n",
+    "def format_docs(docs):\n",
+    "    return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+    "\n",
+    "\n",
+    "# Chain\n",
+    "rag_chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "# Run\n",
+    "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+    "print(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "30d0a69b-9087-4f85-af26-cab55b567872",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'What is the role of memory in artificial intelligence agents?'"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Question Re-writer\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+    "     for web search. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
+    "re_write_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\n",
+    "            \"human\",\n",
+    "            \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
+    "        ),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+    "question_rewriter.invoke({\"question\": question})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e4538467-4a15-4733-b93c-2b79d4d6bf25",
+   "metadata": {},
+   "source": [
+    "## Web Search Tool"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "46d51b53-54a9-4e0a-9f14-e39998f5b340",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Search\n",
+    "\n",
+    "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+    "\n",
+    "web_search_tool = TavilySearchResults(k=3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "87194a1b-535a-4593-ab95-5736fae176d1",
+   "metadata": {},
+   "source": [
+    "## Create Graph \n",
+    "\n",
+    "Now let's create our graph that will use CRAG\n",
+    "\n",
+    "### Define Graph State"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 39,
+   "id": "94b3945f-ef0f-458d-a443-f763903550b0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List\n",
+    "\n",
+    "from typing_extensions import TypedDict\n",
+    "\n",
+    "\n",
+    "class GraphState(TypedDict):\n",
+    "    \"\"\"\n",
+    "    Represents the state of our graph.\n",
+    "\n",
+    "    Attributes:\n",
+    "        question: question\n",
+    "        generation: LLM generation\n",
+    "        web_search: whether to add search\n",
+    "        documents: list of documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question: str\n",
+    "    generation: str\n",
+    "    web_search: str\n",
+    "    documents: List[str]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "id": "efd639c5-82e2-45e6-a94a-6a4039646ef5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.schema import Document\n",
+    "\n",
+    "\n",
+    "def retrieve(state):\n",
+    "    \"\"\"\n",
+    "    Retrieve documents\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, documents, that contains retrieved documents\n",
+    "    \"\"\"\n",
+    "    print(\"---RETRIEVE---\")\n",
+    "    question = state[\"question\"]\n",
+    "\n",
+    "    # Retrieval\n",
+    "    documents = retriever.get_relevant_documents(question)\n",
+    "    return {\"documents\": documents, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def generate(state):\n",
+    "    \"\"\"\n",
+    "    Generate answer\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, generation, that contains LLM generation\n",
+    "    \"\"\"\n",
+    "    print(\"---GENERATE---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # RAG generation\n",
+    "    generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+    "\n",
+    "\n",
+    "def grade_documents(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the retrieved documents are relevant to the question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with only filtered relevant documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Score each doc\n",
+    "    filtered_docs = []\n",
+    "    web_search = \"No\"\n",
+    "    for d in documents:\n",
+    "        score = retrieval_grader.invoke(\n",
+    "            {\"question\": question, \"document\": d.page_content}\n",
+    "        )\n",
+    "        grade = score.binary_score\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+    "            filtered_docs.append(d)\n",
+    "        else:\n",
+    "            print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+    "            web_search = \"Yes\"\n",
+    "            continue\n",
+    "    return {\"documents\": filtered_docs, \"question\": question, \"web_search\": web_search}\n",
+    "\n",
+    "\n",
+    "def transform_query(state):\n",
+    "    \"\"\"\n",
+    "    Transform the query to produce a better question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates question key with a re-phrased question\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---TRANSFORM QUERY---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Re-write question\n",
+    "    better_question = question_rewriter.invoke({\"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": better_question}\n",
+    "\n",
+    "\n",
+    "def web_search(state):\n",
+    "    \"\"\"\n",
+    "    Web search based on the re-phrased question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with appended web results\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---WEB SEARCH---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Web search\n",
+    "    docs = web_search_tool.invoke({\"query\": question})\n",
+    "    web_results = \"\\n\".join([d[\"content\"] for d in docs])\n",
+    "    web_results = Document(page_content=web_results)\n",
+    "    documents.append(web_results)\n",
+    "\n",
+    "    return {\"documents\": documents, \"question\": question}\n",
+    "\n",
+    "\n",
+    "### Edges\n",
+    "\n",
+    "\n",
+    "def decide_to_generate(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether to generate an answer, or re-generate a question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Binary decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+    "    state[\"question\"]\n",
+    "    web_search = state[\"web_search\"]\n",
+    "    state[\"documents\"]\n",
+    "\n",
+    "    if web_search == \"Yes\":\n",
+    "        # All documents have been filtered check_relevance\n",
+    "        # We will re-generate a new query\n",
+    "        print(\n",
+    "            \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+    "        )\n",
+    "        return \"transform_query\"\n",
+    "    else:\n",
+    "        # We have relevant documents, so generate answer\n",
+    "        print(\"---DECISION: GENERATE---\")\n",
+    "        return \"generate\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fa076e90-7132-4fcf-8507-db5990314c4f",
+   "metadata": {},
+   "source": [
+    "### Compile Graph\n",
+    "\n",
+    "The just follows the flow we outlined in the figure above."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "id": "dedae17a-98c6-474d-90a7-9234b7c8cea0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langgraph.graph import END, StateGraph, START\n",
+    "\n",
+    "workflow = StateGraph(GraphState)\n",
+    "\n",
+    "# Define the nodes\n",
+    "workflow.add_node(\"retrieve\", retrieve)  # retrieve\n",
+    "workflow.add_node(\"grade_documents\", grade_documents)  # grade documents\n",
+    "workflow.add_node(\"generate\", generate)  # generate\n",
+    "workflow.add_node(\"transform_query\", transform_query)  # transform_query\n",
+    "workflow.add_node(\"web_search_node\", web_search)  # web search\n",
+    "\n",
+    "# Build graph\n",
+    "workflow.add_edge(START, \"retrieve\")\n",
+    "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"grade_documents\",\n",
+    "    decide_to_generate,\n",
+    "    {\n",
+    "        \"transform_query\": \"transform_query\",\n",
+    "        \"generate\": \"generate\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"transform_query\", \"web_search_node\")\n",
+    "workflow.add_edge(\"web_search_node\", \"generate\")\n",
+    "workflow.add_edge(\"generate\", END)\n",
+    "\n",
+    "# Compile\n",
+    "app = workflow.compile()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "27ba16a8",
+   "metadata": {},
+   "source": [
+    "## Use the graph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "f5b7c2fe-1fc7-4b76-bf93-ba701a40aa6b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---RETRIEVE---\n",
+      "\"Node 'retrieve':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "\"Node 'grade_documents':\"\n",
+      "'\\n---\\n'\n",
+      "---ASSESS GRADED DOCUMENTS---\n",
+      "---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\n",
+      "---TRANSFORM QUERY---\n",
+      "\"Node 'transform_query':\"\n",
+      "'\\n---\\n'\n",
+      "---WEB SEARCH---\n",
+      "\"Node 'web_search_node':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "\"Node '__end__':\"\n",
+      "'\\n---\\n'\n",
+      "('Agents possess short-term memory, which is utilized for in-context learning, '\n",
+      " 'and long-term memory, allowing them to retain and recall vast amounts of '\n",
+      " 'information over extended periods. Some experts also classify working memory '\n",
+      " 'as a distinct type, although it can be considered a part of short-term '\n",
+      " 'memory in many cases.')\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Run\n",
+    "inputs = {\"question\": \"What are the types of agent memory?\"}\n",
+    "for output in app.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        # Node\n",
+    "        pprint(f\"Node '{key}':\")\n",
+    "        # Optional: print full state at each node\n",
+    "        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+    "    pprint(\"\\n---\\n\")\n",
+    "\n",
+    "# Final generation\n",
+    "pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "41ea1108-f385-4774-962d-db157922e231",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---RETRIEVE---\n",
+      "\"Node 'retrieve':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "\"Node 'grade_documents':\"\n",
+      "'\\n---\\n'\n",
+      "---ASSESS GRADED DOCUMENTS---\n",
+      "---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\n",
+      "---TRANSFORM QUERY---\n",
+      "\"Node 'transform_query':\"\n",
+      "'\\n---\\n'\n",
+      "---WEB SEARCH---\n",
+      "\"Node 'web_search_node':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "\"Node '__end__':\"\n",
+      "'\\n---\\n'\n",
+      "('The AlphaCodium paper functions by proposing a code-oriented iterative flow '\n",
+      " 'that involves repeatedly running and fixing generated code against '\n",
+      " 'input-output tests. Its key mechanisms include generating additional data '\n",
+      " 'like problem reflection and test reasoning to aid the iterative process, as '\n",
+      " 'well as enriching the code generation process. AlphaCodium aims to improve '\n",
+      " 'the performance of Large Language Models on code problems by following a '\n",
+      " 'test-based, multi-stage approach.')\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Run\n",
+    "inputs = {\"question\": \"How does the AlphaCodium paper work?\"}\n",
+    "for output in app.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        # Node\n",
+    "        pprint(f\"Node '{key}':\")\n",
+    "        # Optional: print full state at each node\n",
+    "        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+    "    pprint(\"\\n---\\n\")\n",
+    "\n",
+    "# Final generation\n",
+    "pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a7e44593-1959-4abf-8405-5e23aa9398f5",
+   "metadata": {},
+   "source": [
+    "LangSmith Traces - \n",
+    " \n",
+    "* https://smith.langchain.com/public/f6b1716c-e842-4282-9112-1026b93e246b/r\n",
+    "\n",
+    "* https://smith.langchain.com/public/497c8ed9-d9e2-429e-8ada-e64de3ec26c9/r"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_crag_local.ipynb

@@ -0,0 +1,853 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "b77a7d3b-b28a-4dcf-9f1a-861f2f2c5f6c.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAAA9wAAAHTCAYAAADVmRDhAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAPcoAMABAAAAAEAAAHTAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdB5xkQQAAAHWaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjQ2NzwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj45ODg8L2V4aWY6UGl4ZWxYRGltZW5zaW9uPgogICAgICAgICA8ZXhpZjpVc2VyQ29tbWVudD5TY3JlZW5zaG90PC9leGlmOlVzZXJDb21tZW50PgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "92ddc4f4-f7bf-4e0e-b5a5-5abd8a008b21",
+   "metadata": {},
+   "source": [
+    "# Corrective RAG (CRAG) using local LLMs\n",
+    "\n",
+    "[Corrective-RAG (CRAG)](https://arxiv.org/abs/2401.15884) is a strategy for RAG that incorporates self-reflection / self-grading on retrieved documents. \n",
+    "\n",
+    "The paper follows this general flow:\n",
+    "\n",
+    "* If at least one document exceeds the threshold for `relevance`, then it proceeds to generation\n",
+    "* If all documents fall below the `relevance` threshold or if the grader is unsure, then it uses web search to supplement retrieval\n",
+    "* Before generation, it performs knowledge refinement of the search or retrieved documents\n",
+    "* This partitions the document into `knowledge strips`\n",
+    "* It grades each strip, and filters out irrelevant ones\n",
+    "\n",
+    "We will implement some of these ideas from scratch using [LangGraph](https://langchain-ai.github.io/langgraph/):\n",
+    "\n",
+    "* If *any* documents are irrelevant, we'll supplement retrieval with web search. \n",
+    "* We'll skip the knowledge refinement, but this can be added back as a node if desired. \n",
+    "* We'll use [Tavily Search](https://python.langchain.com/v0.2/docs/integrations/tools/tavily_search/) for web search.\n",
+    "\n",
+    "![Screenshot 2024-06-24 at 3.03.16 PM.png](attachment:b77a7d3b-b28a-4dcf-9f1a-861f2f2c5f6c.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6ba4302f-09d9-4d2a-a18d-a6fd23704850",
+   "metadata": {},
+   "source": [
+    "## Setup\n",
+    "\n",
+    "We'll use [Ollama](https://ollama.ai/) to access a local LLM:\n",
+    "\n",
+    "* Download [Ollama app](https://ollama.ai/).\n",
+    "* Pull your model of choice, e.g.: `ollama pull llama3`\n",
+    "\n",
+    "We'll use [Tavily](https://python.langchain.com/v0.2/docs/integrations/tools/tavily_search/) for web search.\n",
+    "\n",
+    "We'll use a vectorstore with [Nomic local embeddings](https://blog.nomic.ai/posts/nomic-embed-text-v1) or, optionally, OpenAI embeddings.\n",
+    "\n",
+    "\n",
+    "Let's install our required packages and set our API keys:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4a660963-bd3d-4c87-b2e4-b6e432055211",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%capture --no-stderr\n",
+    "%pip install -U langchain_community tiktoken langchainhub scikit-learn langchain langgraph tavily-python  nomic[local] langchain-nomic langchain_openai"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68316ba0-854b-41e1-9af5-1f9e965946e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "\n",
+    "def _set_env(key: str):\n",
+    "    if key not in os.environ:\n",
+    "        os.environ[key] = getpass.getpass(f\"{key}:\")\n",
+    "\n",
+    "\n",
+    "_set_env(\"OPENAI_API_KEY\")\n",
+    "_set_env(\"TAVILY_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "98f863ea",
+   "metadata": {},
+   "source": [
+    "<div class=\"admonition tip\">\n",
+    "    <p class=\"admonition-title\">Set up <a href=\"https://smith.langchain.com\">LangSmith</a> for LangGraph development</p>\n",
+    "    <p style=\"padding-top: 5px;\">\n",
+    "        Sign up for LangSmith to quickly spot issues and improve the performance of your LangGraph projects. LangSmith lets you use trace data to debug, test, and monitor your LLM apps built with LangGraph — read more about how to get started <a href=\"https://docs.smith.langchain.com\">here</a>. \n",
+    "    </p>\n",
+    "</div>    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c059c3a3-7f01-4d46-8289-fde4c1b4155f",
+   "metadata": {},
+   "source": [
+    "### LLM\n",
+    "\n",
+    "You can select from [Ollama LLMs](https://ollama.com/library)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2f4db331-c4d0-4c7c-a9a5-0bebc8a89c6c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "local_llm = \"llama3\"\n",
+    "model_tested = \"llama3-8b\"\n",
+    "metadata = f\"CRAG, {model_tested}\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6e2b6eed-3b3f-44b5-a34a-4ade1e94caf0",
+   "metadata": {},
+   "source": [
+    "## Create Index\n",
+    "\n",
+    "Let's index 3 blog posts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "bb8b789b-475b-4e1b-9c66-03504c837830",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "USER_AGENT environment variable not set, consider setting it to identify your requests.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_community.document_loaders import WebBaseLoader\n",
+    "from langchain_community.vectorstores import SKLearnVectorStore\n",
+    "from langchain_nomic.embeddings import NomicEmbeddings  # local\n",
+    "from langchain_openai import OpenAIEmbeddings  # api\n",
+    "\n",
+    "# List of URLs to load documents from\n",
+    "urls = [\n",
+    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+    "]\n",
+    "\n",
+    "# Load documents from the URLs\n",
+    "docs = [WebBaseLoader(url).load() for url in urls]\n",
+    "docs_list = [item for sublist in docs for item in sublist]\n",
+    "\n",
+    "# Initialize a text splitter with specified chunk size and overlap\n",
+    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+    "    chunk_size=250, chunk_overlap=0\n",
+    ")\n",
+    "\n",
+    "# Split the documents into chunks\n",
+    "doc_splits = text_splitter.split_documents(docs_list)\n",
+    "\n",
+    "# Embedding\n",
+    "\"\"\"\n",
+    "embedding=NomicEmbeddings(\n",
+    "    model=\"nomic-embed-text-v1.5\",\n",
+    "    inference_mode=\"local\",\n",
+    ")\n",
+    "\"\"\"\n",
+    "embedding = OpenAIEmbeddings()\n",
+    "\n",
+    "# Add the document chunks to the \"vector store\"\n",
+    "vectorstore = SKLearnVectorStore.from_documents(\n",
+    "    documents=doc_splits,\n",
+    "    embedding=embedding,\n",
+    ")\n",
+    "retriever = vectorstore.as_retriever(k=4)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "fe7fd10a-f64a-48de-a116-6d5890def1af",
+   "metadata": {},
+   "source": [
+    "## Define Tools"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "0e75c029-6c10-47c7-871c-1f4932b25309",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'score': '1'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Retrieval Grader\n",
+    "\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from langchain_community.chat_models import ChatOllama\n",
+    "from langchain_core.output_parsers import JsonOutputParser\n",
+    "from langchain_mistralai.chat_models import ChatMistralAI\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are a teacher grading a quiz. You will be given: \n",
+    "    1/ a QUESTION\n",
+    "    2/ A FACT provided by the student\n",
+    "    \n",
+    "    You are grading RELEVANCE RECALL:\n",
+    "    A score of 1 means that ANY of the statements in the FACT are relevant to the QUESTION. \n",
+    "    A score of 0 means that NONE of the statements in the FACT are relevant to the QUESTION. \n",
+    "    1 is the highest (best) score. 0 is the lowest score you can give. \n",
+    "    \n",
+    "    Explain your reasoning in a step-by-step manner. Ensure your reasoning and conclusion are correct. \n",
+    "    \n",
+    "    Avoid simply stating the correct answer at the outset.\n",
+    "    \n",
+    "    Question: {question} \\n\n",
+    "    Fact: \\n\\n {documents} \\n\\n\n",
+    "    \n",
+    "    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n",
+    "    Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\n",
+    "    \"\"\",\n",
+    "    input_variables=[\"question\", \"documents\"],\n",
+    ")\n",
+    "\n",
+    "retrieval_grader = prompt | llm | JsonOutputParser()\n",
+    "question = \"agent memory\"\n",
+    "docs = retriever.invoke(question)\n",
+    "doc_txt = docs[1].page_content\n",
+    "print(retrieval_grader.invoke({\"question\": question, \"documents\": doc_txt}))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "dad03302-bd93-43fc-949e-af51a3298cfa",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The document mentions \"memory stream\" which is a long-term memory module that records a comprehensive list of agents' experience in natural language. It also discusses short-term memory and long-term memory, with the latter providing the agent with the capability to retain and recall information over extended periods. Additionally, it mentions planning and reflection mechanisms that enable agents to behave conditioned on past experience.\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Generate\n",
+    "\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are an assistant for question-answering tasks. \n",
+    "    \n",
+    "    Use the following documents to answer the question. \n",
+    "    \n",
+    "    If you don't know the answer, just say that you don't know. \n",
+    "    \n",
+    "    Use three sentences maximum and keep the answer concise:\n",
+    "    Question: {question} \n",
+    "    Documents: {documents} \n",
+    "    Answer: \n",
+    "    \"\"\",\n",
+    "    input_variables=[\"question\", \"documents\"],\n",
+    ")\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, temperature=0)\n",
+    "\n",
+    "# Chain\n",
+    "rag_chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "# Run\n",
+    "generation = rag_chain.invoke({\"documents\": docs, \"question\": question})\n",
+    "print(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "b36a2f36-bc5f-408d-a5e8-3fa203c233f6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Search\n",
+    "\n",
+    "from langchain_community.tools.tavily_search import TavilySearchResults\n",
+    "\n",
+    "web_search_tool = TavilySearchResults(k=3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a3421cf0-9067-43fe-8681-0d3189d15dd3",
+   "metadata": {},
+   "source": [
+    "## Create the Graph \n",
+    "\n",
+    "Here we'll explicitly define the majority of the control flow, only using an LLM to define a single branch point following grading."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "10028794-2fbc-43f9-aa4c-7fe3abd69c1e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/jpeg": "/9j/4AAQSkZJRgABAQAAAQABAAD/4gHYSUNDX1BST0ZJTEUAAQEAAAHIAAAAAAQwAABtbnRyUkdCIFhZWiAH4AABAAEAAAAAAABhY3NwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAA9tYAAQAAAADTLQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlkZXNjAAAA8AAAACRyWFlaAAABFAAAABRnWFlaAAABKAAAABRiWFlaAAABPAAAABR3dHB0AAABUAAAABRyVFJDAAABZAAAAChnVFJDAAABZAAAAChiVFJDAAABZAAAAChjcHJ0AAABjAAAADxtbHVjAAAAAAAAAAEAAAAMZW5VUwAAAAgAAAAcAHMAUgBHAEJYWVogAAAAAAAAb6IAADj1AAADkFhZWiAAAAAAAABimQAAt4UAABjaWFlaIAAAAAAAACSgAAAPhAAAts9YWVogAAAAAAAA9tYAAQAAAADTLXBhcmEAAAAAAAQAAAACZmYAAPKnAAANWQAAE9AAAApbAAAAAAAAAABtbHVjAAAAAAAAAAEAAAAMZW5VUwAAACAAAAAcAEcAbwBvAGcAbABlACAASQBuAGMALgAgADIAMAAxADb/2wBDAAMCAgMCAgMDAwMEAwMEBQgFBQQEBQoHBwYIDAoMDAsKCwsNDhIQDQ4RDgsLEBYQERMUFRUVDA8XGBYUGBIUFRT/2wBDAQMEBAUEBQkFBQkUDQsNFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBT/wAARCAHpAL0DASIAAhEBAxEB/8QAHQABAAIDAQEBAQAAAAAAAAAAAAYHBAUIAwkBAv/EAFcQAAEDBAADAgoEBwwEDQUAAAECAwQABQYRBxIhEzEIFBUWFyJBVZPRMlGU0iM2YXGBkbMzN0JSVnR2kpWhsuFTVHKxJCU1Q2J1d4KWorTD1AlERXPB/8QAGwEBAAMBAQEBAAAAAAAAAAAAAAECAwUEBgf/xAA7EQACAQICBgYHBgcBAAAAAAAAAQIDERNSBBIhMVGRFBVBcaGxBSIyYdHS8DNTYoHB4SNCY3KSosI0/9oADAMBAAIRAxEAPwD6p0pSgFKUoBSlKAUpSgNY5k1nZcU25doKFpJSpKpKAQR3gjdfz51WT3xA+1I+dVRjdmt8mBIdegxnXFTpm1rZSSf+Eud5Iraeb9r92w/gJ+VeCv6QoUKs6Ti3qtreux2OvHQNaKlrbyw/Oqye+IH2pHzp51WT3xA+1I+dV55v2v3bD+An5U837X7th/AT8qw610fJLmi3V34vAsPzqsnviB9qR86edVk98QPtSPnVeeb9r92w/gJ+VPN+1+7YfwE/KnWuj5Jc0OrvxeBYfnVZPfED7Uj5086rJ74gfakfOq8837X7th/AT8qeb9r92w/gJ+VOtdHyS5odXfi8Cw/Oqye+IH2pHzp51WT3xA+1I+dV55v2v3bD+An5U837X7th/AT8qda6PklzQ6u/F4FmwbtBuZWIcyPLKNc3YOpXy77t6PTurLqteHcKPBzu/IjMNR0G2wyUtICQT2snr0qyq6ycZRjOO5pPmjmVaeFNw4ClKUMRSlKAUpSgFKUoBSlKAUpSgKfxT/kt7+fTP/Uu1uK0+Kf8lvfz6Z/6l2tLN4zcP7ZNkQ5mdY1Elx3FNPMP3eOhxtaTpSVJK9ggggg9QRXyGnxctMrWX80vNn1tNpU43fYTKoFK4wQG+IMjEYdkvl3mQ1xm582BFSuLBU+Nt9qorCtcvrEpSoAdSRXsrjlw4QdK4gYsDoHRvUbuPd/Dqts3s15zfPrRkeBWNttwyIhTnVsvbJiTISVgvtSGEq28AOdCRyq0dEKTrVeanT2vXVvAic9nqskvDPjFeMvyvObbcMWuceJZbo9FjzG2mezS22y0sNrAeUtTqitShyp5eVSRsHYrc4hxogZTkarFKsF/xm6KiLnx499hpYMphCkpWpspWobSVJ2lWlDmHSogxi2d4/e+KNotFr7OPlD8i5WvJ25jSUQn1wUNIS40T2m0utJ0UpUNK37NVF+G3Ce+2TiPh97Rw+GNR4lrmW+7TXroxKlypDiG1B91SVkrSVNFIUVFe3OqUgbrZwptN7Fs4+7vM1KaaXv4e83mSeE+5P4KXrOcSxO+Kjs24y4c+5xmURivmCSFJ7cLVyEkkpGjynlKquXE79IySyMzpVmn2J5ZIMO5BoPDXtPZOLTo949b8+qqO18JL/M8EFvh/JYbt+RrsCoJYddSpCH9EhJWglOidDYJ76mNn4u220WuO3nztr4eXlSQU2y73qJ2jjYAHapKXNFBVzge31TvVUnGLTVNbm++3YWhKSac32LmWHSoSeOPDgICzxAxYIJICvLUbRI1sfT/ACj9dbvGs3x3NESF4/f7XfURykPKtkxuQGyd6CuRR1vR1v6jXncJJXaN1KL2Jm/wX8fr7/1ZD/ayasSq7wX8fr7/ANWQ/wBrJqxK+5pfY0/7V5HzOl/byFKUrQ8gpSlAKUpQClKUApSlAKUpQFP4p/yW9/Ppn/qXa2KoMZSipUdoknZJQOtbj0TWxDjymbld46HXVvFtqZpCVLUVK0NdBtRr99FMH3xe/tv+VcrSfRuPXnVVRJSbe59rud2GnU4xSaew0viEX/Vmf6gr2QhLaQlCQlI7gkaAraeimD74vf23/Knopg++L39t/wAq83VD+9XJlun0uDNbStl6KYPvi9/bf8qqLglFm53nXFi1XW93RcTG7+LdADUjkUlnsgrSjr1js99Op/6q5MnrClwZZdeTsZl9QLjSHCOm1JBrb+imD74vf23/ACp6KYPvi9/bf8qdT/1VyZHWFLgzS+IRf9WZ/qCvRphpjfZtob338qQN1tvRTB98Xv7b/lT0UwffF7+2/wCVOqH96uTHT6XBmDgv4/X3/qyH+1k1YlR3GcHhYtNly48iZKkSm22luTHu0ISgqKQOnTqtX66kVd1RUIxgneyS5I49eaqVHNdopSlSYClKUApSlAKUpQClKUApSlAKUpQClKUArnfwYP31fCB/paP2Ca6IrnfwYP31fCB/paP2CaA6IpSlAKUpQClKUApSlAKUpQClKUApSlAKUpQClKUApSlAKUpQCud/Bg/fV8IH+lo/YJq5su4j4nw/8U86MosuN+N8/i/le4Mxe25OXn5O0UObl5k713cw+sVzT4N3GPAbZxN45PzM4xuIxcMlMuG6/d46EyWUxwVONkr0tA0dqGwNGgOuKUpQClKUApSlAKUpQClKUApSlAKUpQClKUApSlAKUqEXDiWl11TVit6ruEnRmOO9hF/OleiXPzpSU/l+q8YSnuLwhKo7RVyb0qtjmmWqOxEsrY/i9o8rX6dD/dX555Zd/q9l/W9WmEsy5no6JW4EC8ObgUON3A64iDGD2SWEKudtKR66+UfhWR7TzoB0PapKPqr5ceDNwXkceeMlixVKHPJynPGbm830LURBBcO/YT0QD/GWmvsJ55Zd/q9l/W9VU8IODTHBDKctv+MwLcibkj/bPokOLU3FTzKX2LASlPI3tW9Ek9E9egphLMuZPRK3A6gpVa+eWXf6vZf1vU88su/1ey/rephLMuY6JW4FlUquWs7yeOeZ+02yYgd6Y8pxpf6OZBB/SRUpxzMYORrcYbS9DntDmchSkcjgG9cye8LT1HrJJHUA6PSqulJK6s17mZTo1KavJG9pSlZGApSlAKUpQClKUApSlAKUpQClKUBX+e3RV4uZx1tREFDQeuPKddqFH1GD/wBFQCise0cqTsKUKwUpCUhKQAANAD2VhsrU7kmUuL/dDcik9OukstBP9wB/TUU443CVaODGdzoMl6FNjWOa8xJjuFtxpaWFlKkqGikggEEdRWlfY1Bblbm1t+uFj6LR4qnRTXeTesO83mBj1skXG5zGLfAjp5nZMhwIQgb11J/KQP01ztm3nHh+I4BbLTfbvcLrmc2PHuFxuN7cYJIjLd7NlwocEUuKAH4NGyBrvPMNFxSwrNLRwL4jM5VNeTZEohyLYx5wP3GQysOhLyXJCmmlLbIKFBC+bRBO+7XnsXdVpOy3HWVatOTW1eTuY8JBN4bhpnqj9mvowpZQFc2uX6SSNb307tVT2eYzKgcQ+FWH2/J8jhWeWm7LmqTd31yZSUttrSlbylFZ0pXQ72kEhJT3iLcTsnvXBzKuILtiu12nJg4NEmxo9znvTG2ZHjLkftwlxShsIbSpR16xCid7NQTKrq71u+Fzp6lc8Yrw34kv3BpDt7lQcfuVvkx58s5c/dH1KcaPYyI24zXYrSvlO21BOj3dBWoxzilk+SwFXAuyhP4dY3OcvkNt1aUTrygOMoacAI50gR3HdHf7s2e/VLDF4o6erGmwvGuydadVFmR19pHlN/TaX9Y+sEdCk9FAkHoa524Q2PiZc5GGZWq6Kft9wQ3Luj8rKHZrM1h1okhuGYqG2VBRSpIbWOXlKTzbJrpKrRk4O8d5eLVSO1E1xG/nJbBGmuNpYldWpLKFcwbeQeVxIPtHMDo+0aPtrc1A+F61CZlTQ/ck3BtQ0NaUqMzzD+4H9NTyvTVioz2dtnzVz5urHUm4rsFKUrEyFKUoBSlKAUpSgFKUoBSlKArXKYKrHmDsggiFeAlSVk+qmShISUfnUhKSP/1r/TrMgsMDKrDcbNdGPGrbcI7kWSxzqR2jS0lKk8ySCNgkbBBq0rtaYl8tz0GcyJEV4AKQSQeh2CCOqVAgEKBBBAIIIFV/PxbILGsiO0MghAgIUlaWpSR/0gohCz+UFP8As/XtKONZp7fM6+jaTFRw6hHMjwDH8uxhOO3i2NT7OhKEojulW2+T6CkrB5kqGuigQfy1gWjhJidkxi649GtIctF15vHmJb7slUnmSEErW4pSyeUAb300NdLine truncated
+      "text/plain": [
+       "<IPython.core.display.Image object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from typing import List\n",
+    "from typing_extensions import TypedDict\n",
+    "from IPython.display import Image, display\n",
+    "from langchain.schema import Document\n",
+    "from langgraph.graph import START, END, StateGraph\n",
+    "\n",
+    "\n",
+    "class GraphState(TypedDict):\n",
+    "    \"\"\"\n",
+    "    Represents the state of our graph.\n",
+    "\n",
+    "    Attributes:\n",
+    "        question: question\n",
+    "        generation: LLM generation\n",
+    "        search: whether to add search\n",
+    "        documents: list of documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question: str\n",
+    "    generation: str\n",
+    "    search: str\n",
+    "    documents: List[str]\n",
+    "    steps: List[str]\n",
+    "\n",
+    "\n",
+    "def retrieve(state):\n",
+    "    \"\"\"\n",
+    "    Retrieve documents\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, documents, that contains retrieved documents\n",
+    "    \"\"\"\n",
+    "    question = state[\"question\"]\n",
+    "    documents = retriever.invoke(question)\n",
+    "    steps = state[\"steps\"]\n",
+    "    steps.append(\"retrieve_documents\")\n",
+    "    return {\"documents\": documents, \"question\": question, \"steps\": steps}\n",
+    "\n",
+    "\n",
+    "def generate(state):\n",
+    "    \"\"\"\n",
+    "    Generate answer\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, generation, that contains LLM generation\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "    generation = rag_chain.invoke({\"documents\": documents, \"question\": question})\n",
+    "    steps = state[\"steps\"]\n",
+    "    steps.append(\"generate_answer\")\n",
+    "    return {\n",
+    "        \"documents\": documents,\n",
+    "        \"question\": question,\n",
+    "        \"generation\": generation,\n",
+    "        \"steps\": steps,\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "def grade_documents(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the retrieved documents are relevant to the question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with only filtered relevant documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "    steps = state[\"steps\"]\n",
+    "    steps.append(\"grade_document_retrieval\")\n",
+    "    filtered_docs = []\n",
+    "    search = \"No\"\n",
+    "    for d in documents:\n",
+    "        score = retrieval_grader.invoke(\n",
+    "            {\"question\": question, \"documents\": d.page_content}\n",
+    "        )\n",
+    "        grade = score[\"score\"]\n",
+    "        if grade == \"yes\":\n",
+    "            filtered_docs.append(d)\n",
+    "        else:\n",
+    "            search = \"Yes\"\n",
+    "            continue\n",
+    "    return {\n",
+    "        \"documents\": filtered_docs,\n",
+    "        \"question\": question,\n",
+    "        \"search\": search,\n",
+    "        \"steps\": steps,\n",
+    "    }\n",
+    "\n",
+    "\n",
+    "def web_search(state):\n",
+    "    \"\"\"\n",
+    "    Web search based on the re-phrased question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with appended web results\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state.get(\"documents\", [])\n",
+    "    steps = state[\"steps\"]\n",
+    "    steps.append(\"web_search\")\n",
+    "    web_results = web_search_tool.invoke({\"query\": question})\n",
+    "    documents.extend(\n",
+    "        [\n",
+    "            Document(page_content=d[\"content\"], metadata={\"url\": d[\"url\"]})\n",
+    "            for d in web_results\n",
+    "        ]\n",
+    "    )\n",
+    "    return {\"documents\": documents, \"question\": question, \"steps\": steps}\n",
+    "\n",
+    "\n",
+    "def decide_to_generate(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether to generate an answer, or re-generate a question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Binary decision for next node to call\n",
+    "    \"\"\"\n",
+    "    search = state[\"search\"]\n",
+    "    if search == \"Yes\":\n",
+    "        return \"search\"\n",
+    "    else:\n",
+    "        return \"generate\"\n",
+    "\n",
+    "\n",
+    "# Graph\n",
+    "workflow = StateGraph(GraphState)\n",
+    "\n",
+    "# Define the nodes\n",
+    "workflow.add_node(\"retrieve\", retrieve)  # retrieve\n",
+    "workflow.add_node(\"grade_documents\", grade_documents)  # grade documents\n",
+    "workflow.add_node(\"generate\", generate)  # generate\n",
+    "workflow.add_node(\"web_search\", web_search)  # web search\n",
+    "\n",
+    "# Build graph\n",
+    "workflow.add_edge(START, \"retrieve\")\n",
+    "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"grade_documents\",\n",
+    "    decide_to_generate,\n",
+    "    {\n",
+    "        \"search\": \"web_search\",\n",
+    "        \"generate\": \"generate\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"web_search\", \"generate\")\n",
+    "workflow.add_edge(\"generate\", END)\n",
+    "\n",
+    "custom_graph = workflow.compile()\n",
+    "\n",
+    "display(Image(custom_graph.get_graph(xray=True).draw_mermaid_png()))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "447d1333-082d-479a-a6fa-0ac0df78bb9d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'response': 'According to the documents, there are two types of agent memory:\\n\\n* Short-term memory (STM): This is a data structure that holds information temporarily and allows the agent to process it when needed.\\n* Long-term memory (LTM): This provides the agent with the capability to retain and recall information over extended periods.\\n\\nThese types of memories allow the agent to learn, reason, and make decisions.',\n",
+       " 'steps': ['retrieve_documents',\n",
+       "  'grade_document_retrieval',\n",
+       "  'web_search',\n",
+       "  'generate_answer']}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import uuid\n",
+    "\n",
+    "\n",
+    "def predict_custom_agent_local_answer(example: dict):\n",
+    "    config = {\"configurable\": {\"thread_id\": str(uuid.uuid4())}}\n",
+    "    state_dict = custom_graph.invoke(\n",
+    "        {\"question\": example[\"input\"], \"steps\": []}, config\n",
+    "    )\n",
+    "    return {\"response\": state_dict[\"generation\"], \"steps\": state_dict[\"steps\"]}\n",
+    "\n",
+    "\n",
+    "example = {\"input\": \"What are the types of agent memory?\"}\n",
+    "response = predict_custom_agent_local_answer(example)\n",
+    "response"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "91325c88-ec77-4c79-8a77-cb2e2842bcd4",
+   "metadata": {},
+   "source": [
+    "Trace: \n",
+    "\n",
+    "https://smith.langchain.com/public/88e7579e-2571-4cf6-98d2-1f9ce3359967/r"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1b80d5da-f698-40d2-a2fb-4eac89e35350",
+   "metadata": {},
+   "source": [
+    "## Evaluation\n",
+    "\n",
+    "Now we've defined two different agent architectures that do roughly the same thing!\n",
+    "\n",
+    "We can evaluate them. See our [conceptual guide](https://docs.smith.langchain.com/concepts/evaluation#agents) for context on agent evaluation.\n",
+    "\n",
+    "### Response\n",
+    "\n",
+    "First, we can assess how well [our agent performs on a set of question-answer pairs](https://docs.smith.langchain.com/tutorials/Developers/agents#response-evaluation).\n",
+    "\n",
+    "We'll create a dataset and save it in LangSmith."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "b83706ac-724b-46b1-9f08-66e6c4fac742",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langsmith import Client\n",
+    "\n",
+    "client = Client()\n",
+    "\n",
+    "# Create a dataset\n",
+    "examples = [\n",
+    "    (\n",
+    "        \"How does the ReAct agent use self-reflection? \",\n",
+    "        \"ReAct integrates reasoning and acting, performing actions - such tools like Wikipedia search API - and then observing / reasoning about the tool outputs.\",\n",
+    "    ),\n",
+    "    (\n",
+    "        \"What are the types of biases that can arise with few-shot prompting?\",\n",
+    "        \"The biases that can arise with few-shot prompting include (1) Majority label bias, (2) Recency bias, and (3) Common token bias.\",\n",
+    "    ),\n",
+    "    (\n",
+    "        \"What are five types of adversarial attacks?\",\n",
+    "        \"Five types of adversarial attacks are (1) Token manipulation, (2) Gradient based attack, (3) Jailbreak prompting, (4) Human red-teaming, (5) Model red-teaming.\",\n",
+    "    ),\n",
+    "    (\n",
+    "        \"Who did the Chicago Bears draft first in the 2024 NFL draft”?\",\n",
+    "        \"The Chicago Bears drafted Caleb Williams first in the 2024 NFL draft.\",\n",
+    "    ),\n",
+    "    (\"Who won the 2024 NBA finals?\", \"The Boston Celtics on the 2024 NBA finals\"),\n",
+    "]\n",
+    "\n",
+    "# Save it\n",
+    "dataset_name = \"Corrective RAG Agent Testing\"\n",
+    "if not client.has_dataset(dataset_name=dataset_name):\n",
+    "    dataset = client.create_dataset(dataset_name=dataset_name)\n",
+    "    inputs, outputs = zip(\n",
+    "        *[({\"input\": text}, {\"output\": label}) for text, label in examples]\n",
+    "    )\n",
+    "    client.create_examples(inputs=inputs, outputs=outputs, dataset_id=dataset.id)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a23f6bc0-2d03-488c-8f4b-747c93876788",
+   "metadata": {},
+   "source": [
+    "Now, we'll use an `LLM as a grader` to compare both agent responses to our ground truth reference answer.\n",
+    "\n",
+    "[Here](https://smith.langchain.com/hub/rlm/rag-answer-vs-reference) is the default prompt that we can use.\n",
+    "\n",
+    "We'll use `gpt-4o` as our LLM grader.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "0a63776c-f9cd-46ce-b8cf-95c066dc5b06",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain import hub\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "# Grade prompt\n",
+    "grade_prompt_answer_accuracy = hub.pull(\"langchain-ai/rag-answer-vs-reference\")\n",
+    "\n",
+    "\n",
+    "def answer_evaluator(run, example) -> dict:\n",
+    "    \"\"\"\n",
+    "    A simple evaluator for RAG answer accuracy\n",
+    "    \"\"\"\n",
+    "\n",
+    "    # Get the question, the ground truth reference answer, RAG chain answer prediction\n",
+    "    input_question = example.inputs[\"input\"]\n",
+    "    reference = example.outputs[\"output\"]\n",
+    "    prediction = run.outputs[\"response\"]\n",
+    "\n",
+    "    # Define an LLM grader\n",
+    "    llm = ChatOpenAI(model=\"gpt-4o\", temperature=0)\n",
+    "    answer_grader = grade_prompt_answer_accuracy | llm\n",
+    "\n",
+    "    # Run evaluator\n",
+    "    score = answer_grader.invoke(\n",
+    "        {\n",
+    "            \"question\": input_question,\n",
+    "            \"correct_answer\": reference,\n",
+    "            \"student_answer\": prediction,\n",
+    "        }\n",
+    "    )\n",
+    "    score = score[\"Score\"]\n",
+    "    return {\"key\": \"answer_v_reference_score\", \"score\": score}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "960f1a01-7f8c-429f-83d0-052cea47b32b",
+   "metadata": {},
+   "source": [
+    "### Trajectory\n",
+    "\n",
+    "Second, [we can assess the list of tool calls](https://docs.smith.langchain.com/tutorials/Developers/agents#trajectory) that each agent makes relative to expected trajectories.\n",
+    "\n",
+    "This evaluates the specific reasoning traces taken by our agents!"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "deb28175-27a1-4afc-9747-2983e87fc881",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langsmith.schemas import Example, Run\n",
+    "\n",
+    "# Reasoning traces that we expect the agents to take\n",
+    "expected_trajectory_1 = [\n",
+    "    \"retrieve_documents\",\n",
+    "    \"grade_document_retrieval\",\n",
+    "    \"web_search\",\n",
+    "    \"generate_answer\",\n",
+    "]\n",
+    "expected_trajectory_2 = [\n",
+    "    \"retrieve_documents\",\n",
+    "    \"grade_document_retrieval\",\n",
+    "    \"generate_answer\",\n",
+    "]\n",
+    "\n",
+    "\n",
+    "def find_tool_calls_react(messages):\n",
+    "    \"\"\"\n",
+    "    Find all tool calls in the messages returned\n",
+    "    \"\"\"\n",
+    "    tool_calls = [\n",
+    "        tc[\"name\"] for m in messages[\"messages\"] for tc in getattr(m, \"tool_calls\", [])\n",
+    "    ]\n",
+    "    return tool_calls\n",
+    "\n",
+    "\n",
+    "def check_trajectory_react(root_run: Run, example: Example) -> dict:\n",
+    "    \"\"\"\n",
+    "    Check if all expected tools are called in exact order and without any additional tool calls.\n",
+    "    \"\"\"\n",
+    "    messages = root_run.outputs[\"messages\"]\n",
+    "    tool_calls = find_tool_calls_react(messages)\n",
+    "    print(f\"Tool calls ReAct agent: {tool_calls}\")\n",
+    "    if tool_calls == expected_trajectory_1 or tool_calls == expected_trajectory_2:\n",
+    "        score = 1\n",
+    "    else:\n",
+    "        score = 0\n",
+    "\n",
+    "    return {\"score\": int(score), \"key\": \"tool_calls_in_exact_order\"}\n",
+    "\n",
+    "\n",
+    "def check_trajectory_custom(root_run: Run, example: Example) -> dict:\n",
+    "    \"\"\"\n",
+    "    Check if all expected tools are called in exact order and without any additional tool calls.\n",
+    "    \"\"\"\n",
+    "    tool_calls = root_run.outputs[\"steps\"]\n",
+    "    print(f\"Tool calls custom agent: {tool_calls}\")\n",
+    "    if tool_calls == expected_trajectory_1 or tool_calls == expected_trajectory_2:\n",
+    "        score = 1\n",
+    "    else:\n",
+    "        score = 0\n",
+    "\n",
+    "    return {\"score\": int(score), \"key\": \"tool_calls_in_exact_order\"}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "909b097d-cda1-45ff-8210-afeb2d18b8ae",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "View the evaluation results for experiment: 'custom-agent-llama3-8b-answer-and-tool-use-d6006159' at:\n",
+      "https://smith.langchain.com/o/1fa8b1f4-fcb9-4072-9aa9-983e35ad61b8/datasets/a8b9273b-ca33-4e2f-9f69-9bbc37f6f51b/compare?selectedSessions=83c60822-ef22-43e8-ac85-4488af279c6f\n",
+      "\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "529952314cd34ac1bb115840536921c3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "0it [00:00, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n",
+      "Tool calls custom agent: ['retrieve_documents', 'grade_document_retrieval', 'web_search', 'generate_answer']\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langsmith.evaluation import evaluate\n",
+    "\n",
+    "experiment_prefix = f\"custom-agent-{model_tested}\"\n",
+    "experiment_results = evaluate(\n",
+    "    predict_custom_agent_local_answer,\n",
+    "    data=dataset_name,\n",
+    "    evaluators=[answer_evaluator, check_trajectory_custom],\n",
+    "    experiment_prefix=experiment_prefix + \"-answer-and-tool-use\",\n",
+    "    num_repetitions=3,\n",
+    "    max_concurrency=1,  # Use when running locally\n",
+    "    metadata={\"version\": metadata},\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {
+    "80e86604-7734-4aeb-a200-d1413870c3cb.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABf0AAAFdCAYAAAC5CcfkAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAX9oAMABAAAAAEAAAFdAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdGE8+poAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjM0OTwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNTMzPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "47c38cd3-2c31-48c4-8281-bd9e1f7b2830",
+   "metadata": {},
+   "source": [
+    "We can see the results benchmarked against `GPT-4o` and `Llama-3-70b` using `Custom` agent (as shown here) and ReAct.\n",
+    "\n",
+    "![Screenshot 2024-06-24 at 4.14.04 PM.png](attachment:80e86604-7734-4aeb-a200-d1413870c3cb.png)\n",
+    "\n",
+    "The `local custom agent` performs well in terms of tool calling reliability: it follows the expected reasoning traces.\n",
+    "\n",
+    "However, the answer accuracy performance lags the larger models with `custom agent` implementations."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_self_rag.ipynb

@@ -0,0 +1,792 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "15cba0ab-a549-4909-8373-fb761e384eff.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABpwAAAJ0CAYAAAAPhYDIAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAacoAMABAAAAAEAAAJ0AAAAAEFTQ0lJAAAAU2NyZWVuc2hvdHAfBRUAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjYyODwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNjkyPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "919fe33c-0149-4f7d-b200-544a18986c9a",
+   "metadata": {},
+   "source": [
+    "# Self-RAG\n",
+    "\n",
+    "Self-RAG is a strategy for RAG that incorporates self-reflection / self-grading on retrieved documents and generations. \n",
+    "\n",
+    "In the [paper](https://arxiv.org/abs/2310.11511), a few decisions are made:\n",
+    "\n",
+    "1. Should I retrieve from retriever, `R` -\n",
+    "\n",
+    "* Input: `x (question)` OR `x (question)`, `y (generation)`\n",
+    "* Decides when to retrieve `D` chunks with `R`\n",
+    "* Output: `yes, no, continue`\n",
+    "\n",
+    "2. Are the retrieved passages `D` relevant to the question `x` -\n",
+    "\n",
+    "* * Input: (`x (question)`, `d (chunk)`) for `d` in `D`\n",
+    "* `d` provides useful information to solve `x`\n",
+    "* Output: `relevant, irrelevant`\n",
+    "\n",
+    "3. Are the LLM generation from each chunk in `D` is relevant to the chunk (hallucinations, etc)  -\n",
+    "\n",
+    "* Input: `x (question)`, `d (chunk)`,  `y (generation)` for `d` in `D`\n",
+    "* All of the verification-worthy statements in `y (generation)` are supported by `d`\n",
+    "* Output: `{fully supported, partially supported, no support`\n",
+    "\n",
+    "4. The LLM generation from each chunk in `D` is a useful response to `x (question)` -\n",
+    "\n",
+    "* Input: `x (question)`, `y (generation)` for `d` in `D`\n",
+    "* `y (generation)` is a useful response to `x (question)`.\n",
+    "* Output: `{5, 4, 3, 2, 1}`\n",
+    "\n",
+    "We will implement some of these ideas from scratch using [LangGraph](https://langchain-ai.github.io/langgraph/).\n",
+    "\n",
+    "![Screenshot 2024-04-01 at 12.41.50 PM.png](attachment:15cba0ab-a549-4909-8373-fb761e384eff.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "72f3ee57-68ab-4040-bd36-4014e2a23d96",
+   "metadata": {},
+   "source": [
+    "## Setup\n",
+    "\n",
+    "First let's install our required packages and set our API keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a384cc48-0425-4e8f-aafc-cfb8e56025c9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install -U langchain_community tiktoken langchain-openai langchainhub chromadb langchain langgraph"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "de4ee2a5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "\n",
+    "def _set_env(key: str):\n",
+    "    if key not in os.environ:\n",
+    "        os.environ[key] = getpass.getpass(f\"{key}:\")\n",
+    "\n",
+    "\n",
+    "_set_env(\"OPENAI_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "25d16369",
+   "metadata": {},
+   "source": [
+    "<div class=\"admonition tip\">\n",
+    "    <p class=\"admonition-title\">Set up <a href=\"https://smith.langchain.com\">LangSmith</a> for LangGraph development</p>\n",
+    "    <p style=\"padding-top: 5px;\">\n",
+    "        Sign up for LangSmith to quickly spot issues and improve the performance of your LangGraph projects. LangSmith lets you use trace data to debug, test, and monitor your LLM apps built with LangGraph — read more about how to get started <a href=\"https://docs.smith.langchain.com\">here</a>. \n",
+    "    </p>\n",
+    "</div>    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c27bebdc-be71-4130-ab9d-42f09f87658b",
+   "metadata": {},
+   "source": [
+    "## Retriever\n",
+    " \n",
+    "Let's index 3 blog posts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "565a6d44-2c9f-4fff-b1ec-eea05df9350d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_community.document_loaders import WebBaseLoader\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "\n",
+    "urls = [\n",
+    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+    "]\n",
+    "\n",
+    "docs = [WebBaseLoader(url).load() for url in urls]\n",
+    "docs_list = [item for sublist in docs for item in sublist]\n",
+    "\n",
+    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+    "    chunk_size=250, chunk_overlap=0\n",
+    ")\n",
+    "doc_splits = text_splitter.split_documents(docs_list)\n",
+    "\n",
+    "# Add to vectorDB\n",
+    "vectorstore = Chroma.from_documents(\n",
+    "    documents=doc_splits,\n",
+    "    collection_name=\"rag-chroma\",\n",
+    "    embedding=OpenAIEmbeddings(),\n",
+    ")\n",
+    "retriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "29c12f74-53e2-43cc-896f-875d1c5d9d93",
+   "metadata": {},
+   "source": [
+    "## LLMs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "1fafad21-60cc-483e-92a3-6a7edb1838e3",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/rlm/miniforge3/envs/llama2/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:119: LangChainDeprecationWarning: The method `BaseRetriever.get_relevant_documents` was deprecated in langchain-core 0.1.46 and will be removed in 0.3.0. Use invoke instead.\n",
+      "  warn_deprecated(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "binary_score='yes'\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Retrieval Grader\n",
+    "\n",
+    "\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeDocuments(BaseModel):\n",
+    "    \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+    "    It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
+    "    If the document contains keyword(s) or semantic meaning related to the user question, grade it as relevant. \\n\n",
+    "    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question.\"\"\"\n",
+    "grade_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\"human\", \"Retrieved document: \\n\\n {document} \\n\\n User question: {question}\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "retrieval_grader = grade_prompt | structured_llm_grader\n",
+    "question = \"agent memory\"\n",
+    "docs = retriever.get_relevant_documents(question)\n",
+    "doc_txt = docs[1].page_content\n",
+    "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "dcd77cc1-4587-40ec-b633-5364eab9e1ec",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The design of generative agents combines LLM with memory, planning, and reflection mechanisms to enable agents to behave conditioned on past experience and interact with other agents. Long-term memory provides the agent with the capability to retain and recall infinite information over extended periods. Short-term memory is utilized for in-context learning.\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Generate\n",
+    "\n",
+    "from langchain import hub\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",
+    "\n",
+    "\n",
+    "# Post-processing\n",
+    "def format_docs(docs):\n",
+    "    return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+    "\n",
+    "\n",
+    "# Chain\n",
+    "rag_chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "# Run\n",
+    "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+    "print(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "e78931ec-940c-46ad-a0b2-f43f953f1fd7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GradeHallucinations(binary_score='yes')"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Hallucination Grader\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeHallucinations(BaseModel):\n",
+    "    \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeHallucinations)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You are a grader assessing whether an LLM generation is grounded in / supported by a set of retrieved facts. \\n \n",
+    "     Give a binary score 'yes' or 'no'. 'Yes' means that the answer is grounded in / supported by the set of facts.\"\"\"\n",
+    "hallucination_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\"human\", \"Set of facts: \\n\\n {documents} \\n\\n LLM generation: {generation}\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "hallucination_grader = hallucination_prompt | structured_llm_grader\n",
+    "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "bd62276f-bf26-40d0-8cff-e07b10e00321",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "GradeAnswer(binary_score='yes')"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Answer Grader\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeAnswer(BaseModel):\n",
+    "    \"\"\"Binary score to assess answer addresses question.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Answer addresses the question, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeAnswer)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You are a grader assessing whether an answer addresses / resolves a question \\n \n",
+    "     Give a binary score 'yes' or 'no'. Yes' means that the answer resolves the question.\"\"\"\n",
+    "answer_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\"human\", \"User question: \\n\\n {question} \\n\\n LLM generation: {generation}\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "answer_grader = answer_prompt | structured_llm_grader\n",
+    "answer_grader.invoke({\"question\": question, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "c6f4c70e-1660-4149-82c0-837f19fc9fb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"What is the role of memory in an agent's functioning?\""
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Question Re-writer\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "system = \"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+    "     for vectorstore retrieval. Look at the input and try to reason about the underlying semantic intent / meaning.\"\"\"\n",
+    "re_write_prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", system),\n",
+    "        (\n",
+    "            \"human\",\n",
+    "            \"Here is the initial question: \\n\\n {question} \\n Formulate an improved question.\",\n",
+    "        ),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+    "question_rewriter.invoke({\"question\": question})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "276001c5-c079-4e5b-9f42-81a06704d200",
+   "metadata": {},
+   "source": [
+    "# Graph \n",
+    "\n",
+    "Capture the flow in as a graph.\n",
+    "\n",
+    "## Graph state"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "f1617e9e-66a8-4c1a-a1fe-cc936284c085",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List\n",
+    "\n",
+    "from typing_extensions import TypedDict\n",
+    "\n",
+    "\n",
+    "class GraphState(TypedDict):\n",
+    "    \"\"\"\n",
+    "    Represents the state of our graph.\n",
+    "\n",
+    "    Attributes:\n",
+    "        question: question\n",
+    "        generation: LLM generation\n",
+    "        documents: list of documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question: str\n",
+    "    generation: str\n",
+    "    documents: List[str]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "add509d8-6682-4127-8d95-13dd37d79702",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Nodes\n",
+    "\n",
+    "\n",
+    "def retrieve(state):\n",
+    "    \"\"\"\n",
+    "    Retrieve documents\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, documents, that contains retrieved documents\n",
+    "    \"\"\"\n",
+    "    print(\"---RETRIEVE---\")\n",
+    "    question = state[\"question\"]\n",
+    "\n",
+    "    # Retrieval\n",
+    "    documents = retriever.get_relevant_documents(question)\n",
+    "    return {\"documents\": documents, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def generate(state):\n",
+    "    \"\"\"\n",
+    "    Generate answer\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, generation, that contains LLM generation\n",
+    "    \"\"\"\n",
+    "    print(\"---GENERATE---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # RAG generation\n",
+    "    generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+    "\n",
+    "\n",
+    "def grade_documents(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the retrieved documents are relevant to the question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with only filtered relevant documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Score each doc\n",
+    "    filtered_docs = []\n",
+    "    for d in documents:\n",
+    "        score = retrieval_grader.invoke(\n",
+    "            {\"question\": question, \"document\": d.page_content}\n",
+    "        )\n",
+    "        grade = score.binary_score\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+    "            filtered_docs.append(d)\n",
+    "        else:\n",
+    "            print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+    "            continue\n",
+    "    return {\"documents\": filtered_docs, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def transform_query(state):\n",
+    "    \"\"\"\n",
+    "    Transform the query to produce a better question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates question key with a re-phrased question\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---TRANSFORM QUERY---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Re-write question\n",
+    "    better_question = question_rewriter.invoke({\"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": better_question}\n",
+    "\n",
+    "\n",
+    "### Edges\n",
+    "\n",
+    "\n",
+    "def decide_to_generate(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether to generate an answer, or re-generate a question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Binary decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+    "    state[\"question\"]\n",
+    "    filtered_documents = state[\"documents\"]\n",
+    "\n",
+    "    if not filtered_documents:\n",
+    "        # All documents have been filtered check_relevance\n",
+    "        # We will re-generate a new query\n",
+    "        print(\n",
+    "            \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+    "        )\n",
+    "        return \"transform_query\"\n",
+    "    else:\n",
+    "        # We have relevant documents, so generate answer\n",
+    "        print(\"---DECISION: GENERATE---\")\n",
+    "        return \"generate\"\n",
+    "\n",
+    "\n",
+    "def grade_generation_v_documents_and_question(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the generation is grounded in the document and answers question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK HALLUCINATIONS---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "    generation = state[\"generation\"]\n",
+    "\n",
+    "    score = hallucination_grader.invoke(\n",
+    "        {\"documents\": documents, \"generation\": generation}\n",
+    "    )\n",
+    "    grade = score.binary_score\n",
+    "\n",
+    "    # Check hallucination\n",
+    "    if grade == \"yes\":\n",
+    "        print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
+    "        # Check question-answering\n",
+    "        print(\"---GRADE GENERATION vs QUESTION---\")\n",
+    "        score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
+    "        grade = score.binary_score\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
+    "            return \"useful\"\n",
+    "        else:\n",
+    "            print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
+    "            return \"not useful\"\n",
+    "    else:\n",
+    "        pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
+    "        return \"not supported\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "61cd5797-1782-4d78-a277-8196d13f3e1b",
+   "metadata": {},
+   "source": [
+    "## Build Graph\n",
+    "\n",
+    "The just follows the flow we outlined in the figure above."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "0e09ca9f-e36d-4ef4-a0d5-79fdbada9fe0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langgraph.graph import END, StateGraph, START\n",
+    "\n",
+    "workflow = StateGraph(GraphState)\n",
+    "\n",
+    "# Define the nodes\n",
+    "workflow.add_node(\"retrieve\", retrieve)  # retrieve\n",
+    "workflow.add_node(\"grade_documents\", grade_documents)  # grade documents\n",
+    "workflow.add_node(\"generate\", generate)  # generate\n",
+    "workflow.add_node(\"transform_query\", transform_query)  # transform_query\n",
+    "\n",
+    "# Build graph\n",
+    "workflow.add_edge(START, \"retrieve\")\n",
+    "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"grade_documents\",\n",
+    "    decide_to_generate,\n",
+    "    {\n",
+    "        \"transform_query\": \"transform_query\",\n",
+    "        \"generate\": \"generate\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"transform_query\", \"retrieve\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"generate\",\n",
+    "    grade_generation_v_documents_and_question,\n",
+    "    {\n",
+    "        \"not supported\": \"generate\",\n",
+    "        \"useful\": END,\n",
+    "        \"not useful\": \"transform_query\",\n",
+    "    },\n",
+    ")\n",
+    "\n",
+    "# Compile\n",
+    "app = workflow.compile()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "fb69dbb9-91ee-4868-8c3c-93af3cd885be",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---RETRIEVE---\n",
+      "\"Node 'retrieve':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---ASSESS GRADED DOCUMENTS---\n",
+      "---DECISION: GENERATE---\n",
+      "\"Node 'grade_documents':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "('Short-term memory is used for in-context learning in agents, allowing them '\n",
+      " 'to learn quickly. Long-term memory enables agents to retain and recall vast '\n",
+      " 'amounts of information over extended periods. Agents can also utilize '\n",
+      " 'external tools like APIs to access additional information beyond what is '\n",
+      " 'stored in their memory.')\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Run\n",
+    "inputs = {\"question\": \"Explain how the different types of agent memory work?\"}\n",
+    "for output in app.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        # Node\n",
+    "        pprint(f\"Node '{key}':\")\n",
+    "        # Optional: print full state at each node\n",
+    "        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+    "    pprint(\"\\n---\\n\")\n",
+    "\n",
+    "# Final generation\n",
+    "pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "4138bc51-8c84-4b8a-8d24-f7f470721f6f",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---RETRIEVE---\n",
+      "\"Node 'retrieve':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---ASSESS GRADED DOCUMENTS---\n",
+      "---DECISION: GENERATE---\n",
+      "\"Node 'grade_documents':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "('Chain of thought prompting works by repeatedly prompting the model to ask '\n",
+      " 'follow-up questions to construct the thought process iteratively. This '\n",
+      " 'method can be combined with queries to search for relevant entities and '\n",
+      " 'content to add back into the context. It extends the thought process by '\n",
+      " 'exploring multiple reasoning possibilities at each step, creating a tree '\n",
+      " 'structure of thoughts.')\n"
+     ]
+    }
+   ],
+   "source": [
+    "inputs = {\"question\": \"Explain how chain of thought prompting works?\"}\n",
+    "for output in app.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        # Node\n",
+    "        pprint(f\"Node '{key}':\")\n",
+    "        # Optional: print full state at each node\n",
+    "        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+    "    pprint(\"\\n---\\n\")\n",
+    "\n",
+    "# Final generation\n",
+    "pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "548f1c5b-4108-4aae-8abb-ec171b511b92",
+   "metadata": {},
+   "source": [
+    "LangSmith Traces - \n",
+    " \n",
+    "* https://smith.langchain.com/public/55d6180f-aab8-42bc-8799-dadce6247d9b/r\n",
+    "\n",
+    "* https://smith.langchain.com/public/1c6bf654-61b2-4fc5-9889-054b020c78aa/r"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_self_rag_local.ipynb

@@ -0,0 +1,747 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "5fca0a3e-d13d-4bfa-95ea-58203640cc7a.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABo8AAALNCAYAAAD+2fieAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAaPoAMABAAAAAEAAALNAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdCf/bgEAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjcxNzwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNjc5PC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "848ba742-7443-4123-8115-061da9823309",
+   "metadata": {},
+   "source": [
+    "# Self-RAG using local LLMs\n",
+    "\n",
+    "Self-RAG is a strategy for RAG that incorporates self-reflection / self-grading on retrieved documents and generations. \n",
+    "\n",
+    "In the [paper](https://arxiv.org/abs/2310.11511), a few decisions are made:\n",
+    "\n",
+    "1. Should I retrieve from retriever, `R` -\n",
+    "\n",
+    "* Input: `x (question)` OR `x (question)`, `y (generation)`\n",
+    "* Decides when to retrieve `D` chunks with `R`\n",
+    "* Output: `yes, no, continue`\n",
+    "\n",
+    "2. Are the retrieved passages `D` relevant to the question `x` -\n",
+    "\n",
+    "* * Input: (`x (question)`, `d (chunk)`) for `d` in `D`\n",
+    "* `d` provides useful information to solve `x`\n",
+    "* Output: `relevant, irrelevant`\n",
+    "\n",
+    "3. Are the LLM generation from each chunk in `D` is relevant to the chunk (hallucinations, etc)  -\n",
+    "\n",
+    "* Input: `x (question)`, `d (chunk)`,  `y (generation)` for `d` in `D`\n",
+    "* All of the verification-worthy statements in `y (generation)` are supported by `d`\n",
+    "* Output: `{fully supported, partially supported, no support`\n",
+    "\n",
+    "4. The LLM generation from each chunk in `D` is a useful response to `x (question)` -\n",
+    "\n",
+    "* Input: `x (question)`, `y (generation)` for `d` in `D`\n",
+    "* `y (generation)` is a useful response to `x (question)`.\n",
+    "* Output: `{5, 4, 3, 2, 1}`\n",
+    "\n",
+    "We will implement some of these ideas from scratch using [LangGraph](https://langchain-ai.github.io/langgraph/).\n",
+    "\n",
+    "![Screenshot 2024-04-01 at 12.42.59 PM.png](attachment:5fca0a3e-d13d-4bfa-95ea-58203640cc7a.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9ed0a85a-a33b-40a6-99fa-2444bf57a6cc",
+   "metadata": {},
+   "source": [
+    "## Setup\n",
+    "\n",
+    "First let's install our required packages and set our API keys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d7f9cc6d-a70c-433a-b0ad-ea47c5a0717e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%capture --no-stderr\n",
+    "%pip install -U langchain-nomic langchain_community tiktoken langchainhub chromadb langchain langgraph nomic[local]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "71c540ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "\n",
+    "def _set_env(key: str):\n",
+    "    if key not in os.environ:\n",
+    "        os.environ[key] = getpass.getpass(f\"{key}:\")\n",
+    "\n",
+    "\n",
+    "_set_env(\"NOMIC_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "05e8cf60",
+   "metadata": {},
+   "source": [
+    "<div class=\"admonition tip\">\n",
+    "    <p class=\"admonition-title\">Set up <a href=\"https://smith.langchain.com\">LangSmith</a> for LangGraph development</p>\n",
+    "    <p style=\"padding-top: 5px;\">\n",
+    "        Sign up for LangSmith to quickly spot issues and improve the performance of your LangGraph projects. LangSmith lets you use trace data to debug, test, and monitor your LLM apps built with LangGraph — read more about how to get started <a href=\"https://docs.smith.langchain.com\">here</a>. \n",
+    "    </p>\n",
+    "</div>    "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ccc6b6bd-a2fa-4a43-83b7-704b8b6fb855",
+   "metadata": {},
+   "source": [
+    "### LLMs\n",
+    "\n",
+    "#### Local Embeddings\n",
+    "\n",
+    "You can use `GPT4AllEmbeddings()` from Nomic, which can access use Nomic's recently released [v1](https://blog.nomic.ai/posts/nomic-embed-text-v1) and [v1.5](https://blog.nomic.ai/posts/nomic-embed-matryoshka) embeddings.\n",
+    "\n",
+    "\n",
+    "Follow the documentation [here](https://docs.gpt4all.io/gpt4all_python_embedding.html#supported-embedding-models).\n",
+    "\n",
+    "#### Local LLM\n",
+    "\n",
+    "(1) Download [Ollama app](https://ollama.ai/).\n",
+    "\n",
+    "(2) Download a `Mistral` model from various Mistral versions [here](https://ollama.ai/library/mistral) and Mixtral versions [here](https://ollama.ai/library/mixtral) available.\n",
+    "```\n",
+    "ollama pull mistral\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "bedffc73-6b10-42c8-8768-2085c8ed3398",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Ollama model name\n",
+    "local_llm = \"mistral\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ba68a46d-b617-4fdc-9113-fabdcf736feb",
+   "metadata": {},
+   "source": [
+    "## Create Index\n",
+    "\n",
+    "Let's index 3 blog posts."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c3bb9060-ad74-4470-9991-2ba167b6b8d8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain_community.document_loaders import WebBaseLoader\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_nomic.embeddings import NomicEmbeddings\n",
+    "\n",
+    "urls = [\n",
+    "    \"https://lilianweng.github.io/posts/2023-06-23-agent/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/\",\n",
+    "    \"https://lilianweng.github.io/posts/2023-10-25-adv-attack-llm/\",\n",
+    "]\n",
+    "\n",
+    "docs = [WebBaseLoader(url).load() for url in urls]\n",
+    "docs_list = [item for sublist in docs for item in sublist]\n",
+    "\n",
+    "text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+    "    chunk_size=250, chunk_overlap=0\n",
+    ")\n",
+    "doc_splits = text_splitter.split_documents(docs_list)\n",
+    "\n",
+    "# Add to vectorDB\n",
+    "vectorstore = Chroma.from_documents(\n",
+    "    documents=doc_splits,\n",
+    "    collection_name=\"rag-chroma\",\n",
+    "    embedding=NomicEmbeddings(model=\"nomic-embed-text-v1.5\", inference_mode=\"local\"),\n",
+    ")\n",
+    "retriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "cc60ff95-7e12-4004-b18e-a067a2dcc201",
+   "metadata": {},
+   "source": [
+    "## LLMs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "3aad0c60-3208-48fb-af82-0024630b4da1",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'score': 'yes'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Retrieval Grader\n",
+    "\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from langchain_community.chat_models import ChatOllama\n",
+    "from langchain_core.output_parsers import JsonOutputParser\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+    "\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are a grader assessing relevance of a retrieved document to a user question. \\n \n",
+    "    Here is the retrieved document: \\n\\n {document} \\n\\n\n",
+    "    Here is the user question: {question} \\n\n",
+    "    If the document contains keywords related to the user question, grade it as relevant. \\n\n",
+    "    It does not need to be a stringent test. The goal is to filter out erroneous retrievals. \\n\n",
+    "    Give a binary score 'yes' or 'no' score to indicate whether the document is relevant to the question. \\n\n",
+    "    Provide the binary score as a JSON with a single key 'score' and no premable or explanation.\"\"\",\n",
+    "    input_variables=[\"question\", \"document\"],\n",
+    ")\n",
+    "\n",
+    "retrieval_grader = prompt | llm | JsonOutputParser()\n",
+    "question = \"agent memory\"\n",
+    "docs = retriever.get_relevant_documents(question)\n",
+    "doc_txt = docs[1].page_content\n",
+    "print(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e5e45953-248d-492f-af28-d5e80c664c95",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " In an LLM-powered autonomous agent system, the Large Language Model (LLM) functions as the agent's brain. The agent has key components including memory, planning, and reflection mechanisms. The memory component is a long-term memory module that records a comprehensive list of agents’ experience in natural language. It includes a memory stream, which is an external database for storing past experiences. The reflection mechanism synthesizes memories into higher-level inferences over time and guides the agent's future behavior.\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Generate\n",
+    "\n",
+    "from langchain import hub\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = hub.pull(\"rlm/rag-prompt\")\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, temperature=0)\n",
+    "\n",
+    "\n",
+    "# Post-processing\n",
+    "def format_docs(docs):\n",
+    "    return \"\\n\\n\".join(doc.page_content for doc in docs)\n",
+    "\n",
+    "\n",
+    "# Chain\n",
+    "rag_chain = prompt | llm | StrOutputParser()\n",
+    "\n",
+    "# Run\n",
+    "generation = rag_chain.invoke({\"context\": docs, \"question\": question})\n",
+    "print(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "56297862-df87-42a7-ba9d-310926dfb328",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'score': 'yes'}"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Hallucination Grader\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are a grader assessing whether an answer is grounded in / supported by a set of facts. \\n \n",
+    "    Here are the facts:\n",
+    "    \\n ------- \\n\n",
+    "    {documents} \n",
+    "    \\n ------- \\n\n",
+    "    Here is the answer: {generation}\n",
+    "    Give a binary score 'yes' or 'no' score to indicate whether the answer is grounded in / supported by a set of facts. \\n\n",
+    "    Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
+    "    input_variables=[\"generation\", \"documents\"],\n",
+    ")\n",
+    "\n",
+    "hallucination_grader = prompt | llm | JsonOutputParser()\n",
+    "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "e1dd9174-2df6-45b1-8e69-13381f579c39",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'score': 'yes'}"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Answer Grader\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, format=\"json\", temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = PromptTemplate(\n",
+    "    template=\"\"\"You are a grader assessing whether an answer is useful to resolve a question. \\n \n",
+    "    Here is the answer:\n",
+    "    \\n ------- \\n\n",
+    "    {generation} \n",
+    "    \\n ------- \\n\n",
+    "    Here is the question: {question}\n",
+    "    Give a binary score 'yes' or 'no' to indicate whether the answer is useful to resolve a question. \\n\n",
+    "    Provide the binary score as a JSON with a single key 'score' and no preamble or explanation.\"\"\",\n",
+    "    input_variables=[\"generation\", \"question\"],\n",
+    ")\n",
+    "\n",
+    "answer_grader = prompt | llm | JsonOutputParser()\n",
+    "answer_grader.invoke({\"question\": question, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "5216d92b-1ca1-4bcf-a34f-c99ec2766b54",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "' What is agent memory and how can it be effectively utilized in vector database retrieval?'"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Question Re-writer\n",
+    "\n",
+    "# LLM\n",
+    "llm = ChatOllama(model=local_llm, temperature=0)\n",
+    "\n",
+    "# Prompt\n",
+    "re_write_prompt = PromptTemplate(\n",
+    "    template=\"\"\"You a question re-writer that converts an input question to a better version that is optimized \\n \n",
+    "     for vectorstore retrieval. Look at the initial and formulate an improved question. \\n\n",
+    "     Here is the initial question: \\n\\n {question}. Improved question with no preamble: \\n \"\"\",\n",
+    "    input_variables=[\"generation\", \"question\"],\n",
+    ")\n",
+    "\n",
+    "question_rewriter = re_write_prompt | llm | StrOutputParser()\n",
+    "question_rewriter.invoke({\"question\": question})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3d3339b9-5f30-4d54-bfc9-9091c6035955",
+   "metadata": {},
+   "source": [
+    "# Graph \n",
+    "\n",
+    "Capture the flow in as a graph.\n",
+    "\n",
+    "## Graph state"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "90fb1dc6-c482-483a-8441-39965c401beb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List\n",
+    "\n",
+    "from typing_extensions import TypedDict\n",
+    "\n",
+    "\n",
+    "class GraphState(TypedDict):\n",
+    "    \"\"\"\n",
+    "    Represents the state of our graph.\n",
+    "\n",
+    "    Attributes:\n",
+    "        question: question\n",
+    "        generation: LLM generation\n",
+    "        documents: list of documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    question: str\n",
+    "    generation: str\n",
+    "    documents: List[str]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "5324ea49-5745-47b5-a0a5-bf58c8babe46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Nodes\n",
+    "\n",
+    "\n",
+    "def retrieve(state):\n",
+    "    \"\"\"\n",
+    "    Retrieve documents\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, documents, that contains retrieved documents\n",
+    "    \"\"\"\n",
+    "    print(\"---RETRIEVE---\")\n",
+    "    question = state[\"question\"]\n",
+    "\n",
+    "    # Retrieval\n",
+    "    documents = retriever.get_relevant_documents(question)\n",
+    "    return {\"documents\": documents, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def generate(state):\n",
+    "    \"\"\"\n",
+    "    Generate answer\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): New key added to state, generation, that contains LLM generation\n",
+    "    \"\"\"\n",
+    "    print(\"---GENERATE---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # RAG generation\n",
+    "    generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": question, \"generation\": generation}\n",
+    "\n",
+    "\n",
+    "def grade_documents(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the retrieved documents are relevant to the question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates documents key with only filtered relevant documents\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Score each doc\n",
+    "    filtered_docs = []\n",
+    "    for d in documents:\n",
+    "        score = retrieval_grader.invoke(\n",
+    "            {\"question\": question, \"document\": d.page_content}\n",
+    "        )\n",
+    "        grade = score[\"score\"]\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---GRADE: DOCUMENT RELEVANT---\")\n",
+    "            filtered_docs.append(d)\n",
+    "        else:\n",
+    "            print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n",
+    "            continue\n",
+    "    return {\"documents\": filtered_docs, \"question\": question}\n",
+    "\n",
+    "\n",
+    "def transform_query(state):\n",
+    "    \"\"\"\n",
+    "    Transform the query to produce a better question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        state (dict): Updates question key with a re-phrased question\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---TRANSFORM QUERY---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "\n",
+    "    # Re-write question\n",
+    "    better_question = question_rewriter.invoke({\"question\": question})\n",
+    "    return {\"documents\": documents, \"question\": better_question}\n",
+    "\n",
+    "\n",
+    "### Edges\n",
+    "\n",
+    "\n",
+    "def decide_to_generate(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether to generate an answer, or re-generate a question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Binary decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---ASSESS GRADED DOCUMENTS---\")\n",
+    "    state[\"question\"]\n",
+    "    filtered_documents = state[\"documents\"]\n",
+    "\n",
+    "    if not filtered_documents:\n",
+    "        # All documents have been filtered check_relevance\n",
+    "        # We will re-generate a new query\n",
+    "        print(\n",
+    "            \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n",
+    "        )\n",
+    "        return \"transform_query\"\n",
+    "    else:\n",
+    "        # We have relevant documents, so generate answer\n",
+    "        print(\"---DECISION: GENERATE---\")\n",
+    "        return \"generate\"\n",
+    "\n",
+    "\n",
+    "def grade_generation_v_documents_and_question(state):\n",
+    "    \"\"\"\n",
+    "    Determines whether the generation is grounded in the document and answers question.\n",
+    "\n",
+    "    Args:\n",
+    "        state (dict): The current graph state\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Decision for next node to call\n",
+    "    \"\"\"\n",
+    "\n",
+    "    print(\"---CHECK HALLUCINATIONS---\")\n",
+    "    question = state[\"question\"]\n",
+    "    documents = state[\"documents\"]\n",
+    "    generation = state[\"generation\"]\n",
+    "\n",
+    "    score = hallucination_grader.invoke(\n",
+    "        {\"documents\": documents, \"generation\": generation}\n",
+    "    )\n",
+    "    grade = score[\"score\"]\n",
+    "\n",
+    "    # Check hallucination\n",
+    "    if grade == \"yes\":\n",
+    "        print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n",
+    "        # Check question-answering\n",
+    "        print(\"---GRADE GENERATION vs QUESTION---\")\n",
+    "        score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n",
+    "        grade = score[\"score\"]\n",
+    "        if grade == \"yes\":\n",
+    "            print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n",
+    "            return \"useful\"\n",
+    "        else:\n",
+    "            print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n",
+    "            return \"not useful\"\n",
+    "    else:\n",
+    "        print(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n",
+    "        return \"not supported\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bdf3826c-b668-4f0b-bf83-81c40baaaf02",
+   "metadata": {},
+   "source": [
+    "## Build Graph\n",
+    "\n",
+    "This just follows the flow we outlined in the figure above."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "5605dee4-b2df-46ae-a640-cc2ed90c21a6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langgraph.graph import END, StateGraph, START\n",
+    "\n",
+    "workflow = StateGraph(GraphState)\n",
+    "\n",
+    "# Define the nodes\n",
+    "workflow.add_node(\"retrieve\", retrieve)  # retrieve\n",
+    "workflow.add_node(\"grade_documents\", grade_documents)  # grade documents\n",
+    "workflow.add_node(\"generate\", generate)  # generate\n",
+    "workflow.add_node(\"transform_query\", transform_query)  # transform_query\n",
+    "\n",
+    "# Build graph\n",
+    "workflow.add_edge(START, \"retrieve\")\n",
+    "workflow.add_edge(\"retrieve\", \"grade_documents\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"grade_documents\",\n",
+    "    decide_to_generate,\n",
+    "    {\n",
+    "        \"transform_query\": \"transform_query\",\n",
+    "        \"generate\": \"generate\",\n",
+    "    },\n",
+    ")\n",
+    "workflow.add_edge(\"transform_query\", \"retrieve\")\n",
+    "workflow.add_conditional_edges(\n",
+    "    \"generate\",\n",
+    "    grade_generation_v_documents_and_question,\n",
+    "    {\n",
+    "        \"not supported\": \"generate\",\n",
+    "        \"useful\": END,\n",
+    "        \"not useful\": \"transform_query\",\n",
+    "    },\n",
+    ")\n",
+    "\n",
+    "# Compile\n",
+    "app = workflow.compile()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "105ae1b5-6963-4186-bb83-6d6cb96d095f",
+   "metadata": {},
+   "source": [
+    "## Run\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "26a64f7d-0c14-4e31-a67f-63021dee626e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---RETRIEVE---\n",
+      "\"Node 'retrieve':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "\"Node 'grade_documents':\"\n",
+      "'\\n---\\n'\n",
+      "---ASSESS GRADED DOCUMENTS---\n",
+      "---DECISION: GENERATE---\n",
+      "---GENERATE---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node '__end__':\"\n",
+      "'\\n---\\n'\n",
+      "(' In a LLM-powered autonomous agent system, memory is a key component that '\n",
+      " 'enables agents to store and retrieve information. There are different types '\n",
+      " 'of memory in human brains, such as sensory memory which retains impressions '\n",
+      " 'of sensory information for a few seconds, and long-term memory which records '\n",
+      " \"experiences for extended periods (Lil'Log, 2023). In the context of LLM \"\n",
+      " 'agents, memory is often implemented as an external database or memory stream '\n",
+      " \"(Lil'Log, 2023). The agent can consult this memory to inform its behavior \"\n",
+      " 'based on relevance, recency, and importance. Additionally, reflection '\n",
+      " 'mechanisms synthesize memories into higher-level inferences over time and '\n",
+      " \"guide the agent's future behavior (Lil'Log, 2023).\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n",
+    "\n",
+    "# Run\n",
+    "inputs = {\"question\": \"Explain how the different types of agent memory work?\"}\n",
+    "for output in app.stream(inputs):\n",
+    "    for key, value in output.items():\n",
+    "        # Node\n",
+    "        pprint(f\"Node '{key}':\")\n",
+    "        # Optional: print full state at each node\n",
+    "        # pprint.pprint(value[\"keys\"], indent=2, width=80, depth=None)\n",
+    "    pprint(\"\\n---\\n\")\n",
+    "\n",
+    "# Final generation\n",
+    "pprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f9a27907-2611-4791-910b-0d66c59f5cf5",
+   "metadata": {},
+   "source": [
+    "Trace: \n",
+    "\n",
+    "https://smith.langchain.com/public/4163a342-5260-4852-8602-bda3f95177e7/r"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rag/langgraph_self_rag_pinecone_movies.ipynb

@@ -0,0 +1,474 @@
+{
+ "cells": [
+  {
+   "attachments": {
+    "15cba0ab-a549-4909-8373-fb761e384eff.png": {
+     "image/png": "iVBORw0KGgoAAAANSUhEUgAABpwAAAJ0CAYAAAAPhYDIAAAMP2lDQ1BJQ0MgUHJvZmlsZQAASImVVwdYU8kWnluSkEBCCSAgJfQmCEgJICWEFkB6EWyEJEAoMQaCiB1dVHDtYgEbuiqi2AGxI3YWwd4XRRSUdbFgV96kgK77yvfO9829//3nzH/OnDu3DADqp7hicQ6qAUCuKF8SGxLAGJucwiB1AwTggAYIgMDl5YlZ0dERANrg+e/27ib0hnbNQab1z/7/app8QR4PACQa4jR+Hi8X4kMA4JU8sSQfAKKMN5+aL5Zh2IC2BCYI8UIZzlDgShlOU+B9cp/4WDbEzQCoqHG5kgwAaG2QZxTwMqAGrQ9iJxFfKAJAnQGxb27uZD7EqRDbQB8xxDJ9ZtoPOhl/00wb0uRyM4awYi5yUwkU5olzuNP+z3L8b8vNkQ7GsIJNLVMSGiubM6zb7ezJ4TKsBnGvKC0yCmItiD8I+XJ/iFFKpjQ0QeGPGvLy2LBmQBdiJz43MBxiQ4iDRTmREUo+LV0YzIEYrhC0UJjPiYdYD+KFgrygOKXPZsnkWGUstC5dwmYp+QtciTyuLNZDaXYCS6n/OlPAUepjtKLM+CSIKRBbFAgTIyGmQeyYlx0XrvQZXZTJjhz0kUhjZflbQBwrEIUEKPSxgnRJcKzSvzQ3b3C+2OZMISdSiQ/kZ8aHKuqDNfO48vzhXLA2gYiVMKgjyBsbMTgXviAwSDF3rFsgSohT6nwQ5wfEKsbiFHFOtNIfNxPkhMh4M4hd8wrilGPxxHy4IBX6eLo4PzpekSdelMUNi1bkgy8DEYANAgEDSGFLA5NBFhC29tb3witFTzDgAgnIAALgoGQGRyTJe0TwGAeKwJ8QCUDe0LgAea8AFED+6xCrODqAdHlvgXxENngKcS4IBznwWiofJRqKlgieQEb4j+hc2Hgw3xzYZP3/nh9kvzMsyEQoGelgRIb6oCcxiBhIDCUGE21xA9wX98Yj4NEfNheciXsOzuO7P+EpoZ3wmHCD0EG4M0lYLPkpyzGgA+oHK2uR9mMtcCuo6YYH4D5QHSrjurgBcMBdYRwW7gcju0GWrcxbVhXGT9p/m8EPd0PpR3Yio+RhZH+yzc8jaXY0tyEVWa1/rI8i17SherOHen6Oz/6h+nx4Dv/ZE1uIHcTOY6exi9gxrB4wsJNYA9aCHZfhodX1RL66BqPFyvPJhjrCf8QbvLOySuY51Tj1OH1R9OULCmXvaMCeLJ4mEWZk5jNY8IsgYHBEPMcRDBcnF1cAZN8XxevrTYz8u4Hotnzn5v0BgM/JgYGBo9+5sJMA7PeAj/+R75wNE346VAG4cIQnlRQoOFx2IMC3hDp80vSBMTAHNnA+LsAdeAN/EATCQBSIB8lgIsw+E65zCZgKZoC5oASUgWVgNVgPNoGtYCfYAw6AenAMnAbnwGXQBm6Ae3D1dIEXoA+8A58RBCEhVISO6CMmiCVij7ggTMQXCUIikFgkGUlFMhARIkVmIPOQMmQFsh7ZglQj+5EjyGnkItKO3EEeIT3Ia+QTiqFqqDZqhFqhI1EmykLD0Xh0ApqBTkGL0PnoEnQtWoXuRuvQ0+hl9Abagb5A+zGAqWK6mCnmgDExNhaFpWDpmASbhZVi5VgVVos1wvt8DevAerGPOBGn4wzcAa7gUDwB5+FT8Fn4Ynw9vhOvw5vxa/gjvA//RqASDAn2BC8ChzCWkEGYSighlBO2Ew4TzsJnqYvwjkgk6hKtiR7wWUwmZhGnExcTNxD3Ek8R24mdxH4SiaRPsif5kKJIXFI+qYS0jrSbdJJ0ldRF+qCiqmKi4qISrJKiIlIpVilX2aVyQuWqyjOVz2QNsiXZixxF5pOnkZeSt5EbyVfIXeTPFE2KNcWHEk/JosylrKXUUs5S7lPeqKqqmql6qsaoClXnqK5V3ad6QfWR6kc1LTU7NbbaeDWp2hK1HWqn1O6ovaFSqVZUf2oKNZ+6hFpNPUN9SP1Ao9McaRwanzabVkGro12lvVQnq1uqs9Qnqhepl6sfVL+i3qtB1rDSYGtwNWZpVGgc0bil0a9J13TWjNLM1VysuUvzoma3FknLSitIi681X2ur1hmtTjpGN6ez6Tz6PPo2+ll6lzZR21qbo52lXaa9R7tVu09HS8dVJ1GnUKdC57hOhy6ma6XL0c3RXap7QPem7qdhRsNYwwTDFg2rHXZ12Hu94Xr+egK9Ur29ejf0Pukz9IP0s/WX69frPzDADewMYgymGmw0OGvQO1x7uPdw3vDS4QeG3zVEDe0MYw2nG241bDHsNzI2CjESG60zOmPUa6xr7G+cZbzK+IRxjwndxNdEaLLK5KTJc4YOg8XIYaxlNDP6TA1NQ02lpltMW00/m1mbJZgVm+01e2BOMWeap5uvMm8y77MwsRhjMcOixuKuJdmSaZlpucbyvOV7K2urJKsFVvVW3dZ61hzrIusa6/s2VBs/myk2VTbXbYm2TNts2w22bXaonZtdpl2F3RV71N7dXmi/wb59BGGE5wjRiKoRtxzUHFgOBQ41Do8cdR0jHIsd6x1fjrQYmTJy+cjzI785uTnlOG1zuues5RzmXOzc6Pzaxc6F51Lhcn0UdVTwqNmjGka9crV3FbhudL3tRncb47bArcntq7uHu8S91r3Hw8Ij1aPS4xZTmxnNXMy84EnwDPCc7XnM86OXu1e+1wGvv7wdvLO9d3l3j7YeLRi9bXSnj5kP12eLT4cvwzfVd7Nvh5+pH9evyu+xv7k/33+7/zOWLSuLtZv1MsApQBJwOOA924s9k30qEAsMCSwNbA3SCkoIWh/0MNgsOCO4JrgvxC1kesipUEJoeOjy0FscIw6PU83pC/MImxnWHK4WHhe+PvxxhF2EJKJxDDombMzKMfcjLSNFkfVRIIoTtTLqQbR19JToozHEmOiYipinsc6xM2LPx9HjJsXtinsXHxC/NP5egk2CNKEpUT1xfGJ14vukwKQVSR1jR46dOfZyskGyMLkhhZSSmLI9pX9c0LjV47rGu40vGX9zgvWEwgkXJxpMzJl4fJL6JO6kg6mE1KTUXalfuFHcKm5/GietMq2Px+at4b3g+/NX8XsEPoIVgmfpPukr0rszfDJWZvRk+mWWZ/YK2cL1wldZoVmbst5nR2XvyB7IScrZm6uSm5p7RKQlyhY1TzaeXDi5XWwvLhF3TPGasnpKnyRcsj0PyZuQ15CvDX/kW6Q20l+kjwp8CyoKPkxNnHqwULNQVNgyzW7aomnPioKLfpuOT+dNb5phOmPujEczWTO3zEJmpc1qmm0+e/7srjkhc3bOpczNnvt7sVPxiuK385LmNc43mj9nfucvIb/UlNBKJCW3Fngv2LQQXyhc2Lpo1KJ1i76V8ksvlTmVlZd9WcxbfOlX51/X/jqwJH1J61L3pRuXEZeJlt1c7rd85wrNFUUrOleOWVm3irGqdNXb1ZNWXyx3Ld+0hrJGuqZjbcTahnUW65at+7I+c/2NioCKvZWGlYsq32/gb7i60X9j7SajTWWbPm0Wbr69JWRLXZVVVflW4taCrU+3JW47/xvzt+rtBtvLtn/dIdrRsTN2Z3O1R3X1LsNdS2vQGmlNz+7xu9v2BO5pqHWo3bJXd2/ZPrBPuu/5/tT9Nw+EH2g6yDxYe8jyUOVh+uHSOqRuWl1ffWZ9R0NyQ/uRsCNNjd6Nh486Ht1xzPRYxXGd40tPUE7MPzFwsuhk/ynxqd7TGac7myY13Tsz9sz15pjm1rPhZy+cCz535jzr/MkLPheOXfS6eOQS81L9ZffLdS1uLYd/d/v9cKt7a90VjysNbZ5tje2j209c9bt6+lrgtXPXOdcv34i80X4z4ebtW+Nvddzm3+6+k3Pn1d2Cu5/vzblPuF/6QONB+UPDh1V/2P6xt8O94/ijwEctj+Me3+vkdb54kvfkS9f8p9Sn5c9MnlV3u3Qf6wnuaXs+7nnXC/GLz70lf2r+WfnS5uWhv/z/aukb29f1SvJq4PXiN/pvdrx1fdvUH93/8F3uu8/vSz/of9j5kfnx/KekT88+T/1C+rL2q+3Xxm/h3+4P5A4MiLkSrvxXAIMNTU8H4PUOAKjJANDh/owyTrH/kxui2LPKEfhPWLFHlJs7ALXw/z2mF/7d3AJg3za4/YL66uMBiKYCEO8J0FGjhtrgXk2+r5QZEe4DNkd+TctNA//GFHvOH/L++Qxkqq7g5/O/AFFLfCfKufu9AAAAVmVYSWZNTQAqAAAACAABh2kABAAAAAEAAAAaAAAAAAADkoYABwAAABIAAABEoAIABAAAAAEAAAacoAMABAAAAAEAAAJ0AAAAAEFTQ0lJAAAAU2NyZWVuc2hvdHAfBRUAAAHXaVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIj4KICAgICAgICAgPGV4aWY6UGl4ZWxZRGltZW5zaW9uPjYyODwvZXhpZjpQaXhlbFlEaW1lbnNpb24+CiAgICAgICAgIDxleGlmOlBpeGVsWERpbWVuc2lvbj4xNjkyPC9leGlmOlBpeGVsWERpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgPC9yZGY6RGVzY3JpcHRpb24+CiAgIDwvLine truncated
+    }
+   },
+   "cell_type": "markdown",
+   "id": "919fe33c-0149-4f7d-b200-544a18986c9a",
+   "metadata": {},
+   "source": [
+    "# Self RAG\n",
+    "\n",
+    "Self-RAG is a strategy for RAG that incorporates self-reflection / self-grading on retrieved documents and generations. \n",
+    "\n",
+    "[Paper](https://arxiv.org/abs/2310.11511)\n",
+    "\n",
+    "![Screenshot 2024-04-01 at 12.41.50 PM.png](attachment:15cba0ab-a549-4909-8373-fb761e384eff.png)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "72f3ee57-68ab-4040-bd36-4014e2a23d96",
+   "metadata": {},
+   "source": [
+    "# Environment "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a384cc48-0425-4e8f-aafc-cfb8e56025c9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install -qU langchain-pinecone langchain-openai langchainhub langgraph"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "532d91fb-381e-4e11-b3b1-254321351773",
+   "metadata": {},
+   "source": [
+    "### Tracing\n",
+    "\n",
+    "Use [LangSmith](https://docs.smith.langchain.com/) for tracing (shown at bottom)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ccc3dae5-1df6-48ca-af8a-50f0e6128876",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = \"<your-api-key>\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "88637820",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n\nos.environ[\"LANGCHAIN_PROJECT\"] = \"pinecone-devconnect\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c27bebdc-be71-4130-ab9d-42f09f87658b",
+   "metadata": {},
+   "source": [
+    "## Retriever\n",
+    " \n",
+    "Let's use Pinecone's sample movies database"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "565a6d44-2c9f-4fff-b1ec-eea05df9350d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_openai import OpenAIEmbeddings\nfrom langchain_pinecone import PineconeVectorStore\n\n# use pinecone movies database\n\n# Add to vectorDB\nvectorstore = PineconeVectorStore(\n    embedding=OpenAIEmbeddings(),\n    index_name=\"sample-movies\",\n    text_key=\"summary\",\n)\nretriever = vectorstore.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1aeb4373",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "# Avatar\n",
+      "On the alien world of Pandora, paraplegic Marine Jake Sully uses an avatar to walk again and becomes torn between his mission and protecting the planet's indigenous Na'vi people. The film stars Sam Worthington, Zoe Saldana, and Sigourney Weaver.\n",
+      "\n",
+      "# Top Gun: Maverick\n",
+      "Capt. Pete \"Maverick\" Mitchell, after decades of service as one of the Navy's top aviators, confronts his past while training a new squad for a dangerous mission. Tom Cruise reprises his iconic role, showcasing thrilling aerial stunts.\n",
+      "\n",
+      "# Jurassic World Dominion\n",
+      "The film concludes the story of Jurassic World, with humanity now living alongside dinosaurs. It follows Owen Grady and Claire Dearing as they navigate this new world.\n",
+      "\n",
+      "# Aquaman\n",
+      "Arthur Curry learns he is the heir to the underwater kingdom of Atlantis and must step forward to lead his people and be a hero to the world. Stars Jason Momoa, Amber Heard, and Willem Dafoe.\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = retriever.invoke(\"James Cameron\")\nfor doc in docs:\n    print(\"# \" + doc.metadata[\"title\"])\n    print(doc.page_content)\n    print()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "29c12f74-53e2-43cc-896f-875d1c5d9d93",
+   "metadata": {},
+   "source": [
+    "## Structured Output - Retrieval Grader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "1fafad21-60cc-483e-92a3-6a7edb1838e3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Retrieval Grader\n",
+    "\n",
+    "from langchain import hub\n",
+    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeDocuments(BaseModel):\n",
+    "    \"\"\"Binary score for relevance check on retrieved documents.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Documents are relevant to the question, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# https://smith.langchain.com/hub/efriis/self-rag-retrieval-grader\n",
+    "grade_prompt = hub.pull(\"efriis/self-rag-retrieval-grader\")\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeDocuments)\n",
+    "\n",
+    "retrieval_grader = grade_prompt | structured_llm_grader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "2e79eed8",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Arthur Curry learns he is the heir to the underwater kingdom of Atlantis and must step forward to lead his people and be a hero to the world. Stars Jason Momoa, Amber Heard, and Willem Dafoe.\n",
+      "binary_score='yes'\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Test the retrieval grader\nquestion = \"movies starring jason momoa\"\ndocs = retriever.invoke(question)\ndoc_txt = docs[0].page_content\nprint(doc_txt)\nprint(retrieval_grader.invoke({\"question\": question, \"document\": doc_txt}))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c06abc65",
+   "metadata": {},
+   "source": [
+    "# Generation Step\n",
+    "\n",
+    "Standard RAG"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "dcd77cc1-4587-40ec-b633-5364eab9e1ec",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Jason Momoa stars in the movies \"Aquaman\" and \"Furious 7.\"\n"
+     ]
+    }
+   ],
+   "source": [
+    "### Generate\n\nfrom langchain import hub\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Prompt\nprompt = hub.pull(\"rlm/rag-prompt\")\n\n# LLM\nllm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n\n# Chain\nrag_chain = prompt | llm | StrOutputParser()\n\n# Run\ngeneration = rag_chain.invoke({\"context\": docs, \"question\": question})\nprint(generation)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e78931ec-940c-46ad-a0b2-f43f953f1fd7",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Jason Momoa stars in the movies \"Aquaman\" and \"Furious 7.\"\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "GradeHallucinations(binary_score='yes')"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Hallucination Grader\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeHallucinations(BaseModel):\n",
+    "    \"\"\"Binary score for hallucination present in generation answer.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Answer is grounded in the facts, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeHallucinations)\n",
+    "\n",
+    "# https://smith.langchain.com/hub/efriis/self-rag-hallucination-grader\n",
+    "hallucination_prompt = hub.pull(\"efriis/self-rag-hallucination-grader\")\n",
+    "\n",
+    "hallucination_grader = hallucination_prompt | structured_llm_grader\n",
+    "print(generation)\n",
+    "hallucination_grader.invoke({\"documents\": docs, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "bd62276f-bf26-40d0-8cff-e07b10e00321",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "movies starring jason momoa\n",
+      "Jason Momoa stars in the movies \"Aquaman\" and \"Furious 7.\"\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "GradeAnswer(binary_score='yes')"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Answer Grader\n",
+    "\n",
+    "\n",
+    "# Data model\n",
+    "class GradeAnswer(BaseModel):\n",
+    "    \"\"\"Binary score to assess answer addresses question.\"\"\"\n",
+    "\n",
+    "    binary_score: str = Field(\n",
+    "        description=\"Answer addresses the question, 'yes' or 'no'\"\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "# LLM with function call\n",
+    "llm = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n",
+    "structured_llm_grader = llm.with_structured_output(GradeAnswer)\n",
+    "\n",
+    "# Prompt\n",
+    "answer_prompt = hub.pull(\"efriis/self-rag-answer-grader\")\n",
+    "\n",
+    "answer_grader = answer_prompt | structured_llm_grader\n",
+    "print(question)\n",
+    "print(generation)\n",
+    "answer_grader.invoke({\"question\": question, \"generation\": generation})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "c6f4c70e-1660-4149-82c0-837f19fc9fb5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "movies starring jason momoa\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'Which movies feature Jason Momoa in a leading role?'"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "### Question Re-writer\n\n# LLM\nllm = ChatOpenAI(model=\"gpt-3.5-turbo-0125\", temperature=0)\n\n# Prompt\nre_write_prompt = hub.pull(\"efriis/self-rag-question-rewriter\")\n\nquestion_rewriter = re_write_prompt | llm | StrOutputParser()\nprint(question)\nquestion_rewriter.invoke({\"question\": question})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "276001c5-c079-4e5b-9f42-81a06704d200",
+   "metadata": {},
+   "source": [
+    "# Graph \n",
+    "\n",
+    "Capture the flow in as a graph.\n",
+    "\n",
+    "## Graph state"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "f1617e9e-66a8-4c1a-a1fe-cc936284c085",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List\n\nfrom typing_extensions import TypedDict\n\n\nclass GraphState(TypedDict):\n    \"\"\"\n    Represents the state of our graph.\n\n    Attributes:\n        question: question\n        generation: LLM generation\n        documents: list of documents\n    \"\"\"\n\n    question: str\n    generation: str\n    documents: List[str]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "add509d8-6682-4127-8d95-13dd37d79702",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Nodes\n\n\ndef retrieve(state):\n    \"\"\"\n    Retrieve documents\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): New key added to state, documents, that contains retrieved documents\n    \"\"\"\n    print(\"---RETRIEVE---\")\n    question = state[\"question\"]\n\n    # Retrieval\n    documents = retriever.invoke(question)\n    return {\"documents\": documents, \"question\": question}\n\n\ndef generate(state):\n    \"\"\"\n    Generate answer\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): New key added to state, generation, that contains LLM generation\n    \"\"\"\n    print(\"---GENERATE---\")\n    question = state[\"question\"]\n    documents = state[\"documents\"]\n\n    # RAG generation\n    generation = rag_chain.invoke({\"context\": documents, \"question\": question})\n    return {\"documents\": documents, \"question\": question, \"generation\": generation}\n\n\ndef grade_documents(state):\n    \"\"\"\n    Determines whether the retrieved documents are relevant to the question.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): Updates documents key with only filtered relevant documents\n    \"\"\"\n\n    print(\"---CHECK DOCUMENT RELEVANCE TO QUESTION---\")\n    question = state[\"question\"]\n    documents = state[\"documents\"]\n\n    # Score each doc\n    filtered_docs = []\n    for d in documents:\n        score = retrieval_grader.invoke(\n            {\"question\": question, \"document\": d.page_content}\n        )\n        grade = score.binary_score\n        if grade == \"yes\":\n            print(\"---GRADE: DOCUMENT RELEVANT---\")\n            filtered_docs.append(d)\n        else:\n            print(\"---GRADE: DOCUMENT NOT RELEVANT---\")\n            continue\n    return {\"documents\": filtered_docs, \"question\": question}\n\n\ndef transform_query(state):\n    \"\"\"\n    Transform the query to produce a better question.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        state (dict): Updates question key with a re-phrased question\n    \"\"\"\n\n    print(\"---TRANSFORM QUERY---\")\n    question = state[\"question\"]\n    documents = state[\"documents\"]\n\n    # Re-write question\n    better_question = question_rewriter.invoke({\"question\": question})\n    return {\"documents\": documents, \"question\": better_question}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "09fc91b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Edges\n\n\ndef decide_to_generate(state):\n    \"\"\"\n    Determines whether to generate an answer, or re-generate a question.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        str: Binary decision for next node to call\n    \"\"\"\n\n    print(\"---ASSESS GRADED DOCUMENTS---\")\n    state[\"question\"]\n    filtered_documents = state[\"documents\"]\n\n    if not filtered_documents:\n        # All documents have been filtered check_relevance\n        # We will re-generate a new query\n        print(\n            \"---DECISION: ALL DOCUMENTS ARE NOT RELEVANT TO QUESTION, TRANSFORM QUERY---\"\n        )\n        return \"transform_query\"\n    else:\n        # We have relevant documents, so generate answer\n        print(\"---DECISION: GENERATE---\")\n        return \"generate\"\n\n\ndef grade_generation_v_documents_and_question(state):\n    \"\"\"\n    Determines whether the generation is grounded in the document and answers question.\n\n    Args:\n        state (dict): The current graph state\n\n    Returns:\n        str: Decision for next node to call\n    \"\"\"\n\n    print(\"---CHECK HALLUCINATIONS---\")\n    question = state[\"question\"]\n    documents = state[\"documents\"]\n    generation = state[\"generation\"]\n\n    score = hallucination_grader.invoke(\n        {\"documents\": documents, \"generation\": generation}\n    )\n    grade = score.binary_score\n\n    # Check hallucination\n    if grade == \"yes\":\n        print(\"---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\")\n        # Check question-answering\n        print(\"---GRADE GENERATION vs QUESTION---\")\n        score = answer_grader.invoke({\"question\": question, \"generation\": generation})\n        grade = score.binary_score\n        if grade == \"yes\":\n            print(\"---DECISION: GENERATION ADDRESSES QUESTION---\")\n            return \"useful\"\n        else:\n            print(\"---DECISION: GENERATION DOES NOT ADDRESS QUESTION---\")\n            return \"not useful\"\n    else:\n        pprint(\"---DECISION: GENERATION IS NOT GROUNDED IN DOCUMENTS, RE-TRY---\")\n        return \"not supported\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "61cd5797-1782-4d78-a277-8196d13f3e1b",
+   "metadata": {},
+   "source": [
+    "## Build Graph\n",
+    "\n",
+    "The just follows the flow we outlined in the figure above."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "0e09ca9f-e36d-4ef4-a0d5-79fdbada9fe0",
+   "metadata": {},
+   "outputs": [],
+   "source": ["from langgraph.graph import END, StateGraph, START\n\nworkflow = StateGraph(GraphState)\n\n# Define the nodes\nworkflow.add_node(\"retrieve\", retrieve)  # retrieve\nworkflow.add_node(\"grade_documents\", grade_documents)  # grade documents\nworkflow.add_node(\"generate\", generate)  # generate\nworkflow.add_node(\"transform_query\", transform_query)  # transform_query\n\n# Build graph\nworkflow.add_edge(START, \"retrieve\")\nworkflow.add_edge(\"retrieve\", \"grade_documents\")\nworkflow.add_conditional_edges(\n    \"grade_documents\",\n    decide_to_generate,\n    {\n        \"transform_query\": \"transform_query\",\n        \"generate\": \"generate\",\n    },\n)\nworkflow.add_edge(\"transform_query\", \"retrieve\")\nworkflow.add_conditional_edges(\n    \"generate\",\n    grade_generation_v_documents_and_question,\n    {\n        \"not supported\": \"generate\",\n        \"useful\": END,\n        \"not useful\": \"transform_query\",\n    },\n)\n\n# Compile\napp = workflow.compile()"]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "fb69dbb9-91ee-4868-8c3c-93af3cd885be",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "---RETRIEVE---\n",
+      "\"Node 'retrieve':\"\n",
+      "'\\n---\\n'\n",
+      "---CHECK DOCUMENT RELEVANCE TO QUESTION---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT RELEVANT---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---GRADE: DOCUMENT NOT RELEVANT---\n",
+      "---ASSESS GRADED DOCUMENTS---\n",
+      "---DECISION: GENERATE---\n",
+      "\"Node 'grade_documents':\"\n",
+      "'\\n---\\n'\n",
+      "---GENERATE---\n",
+      "---CHECK HALLUCINATIONS---\n",
+      "---DECISION: GENERATION IS GROUNDED IN DOCUMENTS---\n",
+      "---GRADE GENERATION vs QUESTION---\n",
+      "---DECISION: GENERATION ADDRESSES QUESTION---\n",
+      "\"Node 'generate':\"\n",
+      "'\\n---\\n'\n",
+      "'Daniel Craig stars as 007 in \"Skyfall\" (2012) and \"Spectre\" (2015).'\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pprint import pprint\n\n# Run\ninputs = {\"question\": \"Movies that star Daniel Craig\"}\nfor output in app.stream(inputs):\n    for key, value in output.items():\n        # Node\n        pprint(f\"Node '{key}':\")\n    pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4138bc51-8c84-4b8a-8d24-f7f470721f6f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "inputs = {\"question\": \"Which movies are about aliens?\"}\nfor output in app.stream(inputs):\n    for key, value in output.items():\n        # Node\n        pprint(f\"Node '{key}':\")\n    pprint(\"\\n---\\n\")\n\n# Final generation\npprint(value[\"generation\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "42369ab8-322d-434a-b5dd-2266e4cb2903",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    ""
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/react-agent-from-scratch.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "294995c4",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/react-agent-from-scratch.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

examples/react-agent-structured-output.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "40f0d107",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/react-agent-structured-output.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/recursion-limit.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "fa3f7c50",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/recursion-limit.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

examples/reflection/reflection.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "658773a2",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/reflection/reflection.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/reflexion/reflexion.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "caf07859",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/reflexion/reflexion.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/rewoo/rewoo.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "961f43ec",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/rewoo/rewoo.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/run-id-langsmith.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "bbd6e9b8",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/run-id-langsmith.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/self-discover/self-discover.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f6db1873",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/self-discover/self-discover.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/state-model.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4149ffcc",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/state-model.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/storm/storm.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "3e05d7f9",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/storm/storm.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/stream-multiple.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e663f597",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/stream-multiple.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/stream-updates.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e6829c80",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/stream-updates.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/stream-values.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "5ec11895",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/stream-values.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/streaming-content.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "6619387c",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/streaming-content.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/streaming-events-from-within-tools-without-langchain.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "57b7e303",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/streaming-events-from-within-tools-without-langchain.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/streaming-events-from-within-tools.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "8e71a0c8",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/streaming-events-from-within-tools.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/streaming-from-final-node.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "756e4554",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/streaming-from-final-node.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/streaming-subgraphs.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "47164a72",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/streaming-subgraphs.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

examples/streaming-tokens-without-langchain.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "218dfbcb",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/streaming-tokens-without-langchain.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/streaming-tokens.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "99eb887e",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/streaming-tokens.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/subgraph-transform-state.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0de7689f",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/subgraph-transform-state.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/subgraph.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f49876e1",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/subgraph.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/subgraphs-manage-state.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "5106959e",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/subgraphs-manage-state.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/tool-calling-errors.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "dc21501d",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/tool-calling-errors.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/tool-calling.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7fd8bd65",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/tool-calling.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/tutorials/sql-agent.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "83c2223f",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/sql-agent.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/tutorials/tnt-llm/tnt-llm.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "11140167",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/tnt-llm/tnt-llm.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/usaco/usaco.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9dffdb54",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/usaco/usaco.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/visualization.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9c9cb15a",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/how-tos/visualization.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

examples/web-navigation/web_voyager.ipynb

@@ -0,0 +1,33 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "007ea2e9",
+   "metadata": {},
+   "source": [
+    "This file has been moved to https://github.com/langchain-ai/langgraph/blob/main/docs/docs/tutorials/web-navigation/web_voyager.ipynb"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

libs/checkpoint/langgraph/checkpoint/serde/jsonplus.py

@@ -37,6 +37,48 @@ LC_REVIVER = Reviver()
 EMPTY_BYTES = b""
 logger = logging.getLogger(__name__)
 
+SAFE_MSGPACK_TYPES: frozenset[tuple[str, ...]] = frozenset(
+    {
+        # datetime types
+        ("datetime", "datetime"),
+        ("datetime", "date"),
+        ("datetime", "time"),
+        ("datetime", "timedelta"),
+        ("datetime", "timezone"),
+        # uuid
+        ("uuid", "UUID"),
+        # numeric
+        ("decimal", "Decimal"),
+        # collections
+        ("builtins", "set"),
+        ("builtins", "frozenset"),
+        ("collections", "deque"),
+        # ip addresses
+        ("ipaddress", "IPv4Address"),
+        ("ipaddress", "IPv4Interface"),
+        ("ipaddress", "IPv4Network"),
+        ("ipaddress", "IPv6Address"),
+        ("ipaddress", "IPv6Interface"),
+        ("ipaddress", "IPv6Network"),
+        # pathlib
+        ("pathlib", "Path"),
+        ("pathlib", "PosixPath"),
+        ("pathlib", "WindowsPath"),
+        # pathlib in Python 3.13+
+        ("pathlib._local", "Path"),
+        ("pathlib._local", "PosixPath"),
+        ("pathlib._local", "WindowsPath"),
+        # regex
+        ("re", "compile"),
+        # langgraph
+        ("langgraph.types", "Send"),
+        ("langgraph.types", "Interrupt"),
+        ("langgraph.types", "Command"),
+        ("langgraph.types", "StateSnapshot"),
+        ("langgraph.types", "PregelTask"),
+    }
+)
+
 
 class JsonPlusSerializer(SerializerProtocol):
     """Serializer that uses ormsgpack, with optional fallbacks.
@@ -54,18 +96,29 @@ class JsonPlusSerializer(SerializerProtocol):
         *,
         pickle_fallback: bool = False,
         allowed_json_modules: Sequence[tuple[str, ...]] | Literal[True] | None = None,
+        # TODO: change default to None once users have had time to configure allowlists
+        allowed_msgpack_modules: Sequence[tuple[str, ...]]
+        | Literal[True]
+        | None = True,
         __unpack_ext_hook__: Callable[[int, bytes], Any] | None = None,
     ) -> None:
         self.pickle_fallback = pickle_fallback
-        self._allowed_modules = (
+        # JSON allowlist
+        self._allowed_json_modules: set[tuple[str, ...]] | Literal[True] | None = (
             {mod_and_name for mod_and_name in allowed_json_modules}
             if allowed_json_modules and allowed_json_modules is not True
             else (allowed_json_modules if allowed_json_modules is True else None)
         )
+        # Msgpack allowlist
+        self._allowed_msgpack_modules: set[tuple[str, ...]] | Literal[True] | None = (
+            {mod_and_name for mod_and_name in allowed_msgpack_modules}
+            if allowed_msgpack_modules and allowed_msgpack_modules is not True
+            else (allowed_msgpack_modules if allowed_msgpack_modules is True else None)
+        )
         self._unpack_ext_hook = (
             __unpack_ext_hook__
             if __unpack_ext_hook__ is not None
-            else _msgpack_ext_hook
+            else _create_msgpack_ext_hook(self._allowed_msgpack_modules)
         )
 
     def _encode_constructor_args(
@@ -90,7 +143,7 @@ class JsonPlusSerializer(SerializerProtocol):
         return out
 
     def _reviver(self, value: dict[str, Any]) -> Any:
-        if self._allowed_modules and (
+        if self._allowed_json_modules and (
             value.get("lc", None) == 2
             and value.get("type", None) == "constructor"
             and value.get("id", None) is not None
@@ -107,7 +160,7 @@ class JsonPlusSerializer(SerializerProtocol):
         return LC_REVIVER(value)
 
     def _revive_lc2(self, value: dict[str, Any]) -> Any:
-        self._check_allowed_modules(value)
+        self._check_allowed_json_modules(value)
 
         [*module, name] = value["id"]
         try:
@@ -139,7 +192,7 @@ class JsonPlusSerializer(SerializerProtocol):
         except Exception:
             return None
 
-    def _check_allowed_modules(self, value: dict[str, Any]) -> None:
+    def _check_allowed_json_modules(self, value: dict[str, Any]) -> None:
         needed = tuple(value["id"])
         method = value.get("method")
         if isinstance(method, list):
@@ -150,7 +203,7 @@ class JsonPlusSerializer(SerializerProtocol):
             method_display = "<init>"
 
         dotted = ".".join(needed)
-        if not self._allowed_modules:
+        if not self._allowed_json_modules:
             raise InvalidModuleError(
                 f"Refused to deserialize JSON constructor: {dotted} (method: {method_display}). "
                 "No allowed_json_modules configured.\n\n"
@@ -161,9 +214,9 @@ class JsonPlusSerializer(SerializerProtocol):
                 "or plain-JSON representations revived without import-time side effects."
             )
 
-        if self._allowed_modules is True:
+        if self._allowed_json_modules is True:
             return
-        if needed in self._allowed_modules:
+        if needed in self._allowed_json_modules:
             return
 
         raise InvalidModuleError(
@@ -448,92 +501,158 @@ def _msgpack_default(obj: Any) -> str | ormsgpack.Ext:
         raise TypeError(f"Object of type {obj.__class__.__name__} is not serializable")
 
 
-def _msgpack_ext_hook(code: int, data: bytes) -> Any:
-    if code == EXT_CONSTRUCTOR_SINGLE_ARG:
-        try:
-            tup = ormsgpack.unpackb(
-                data, ext_hook=_msgpack_ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
-            )
-            # module, name, arg
-            return getattr(importlib.import_module(tup[0]), tup[1])(tup[2])
-        except Exception:
-            return
-    elif code == EXT_CONSTRUCTOR_POS_ARGS:
-        try:
-            tup = ormsgpack.unpackb(
-                data, ext_hook=_msgpack_ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
-            )
-            # module, name, args
-            return getattr(importlib.import_module(tup[0]), tup[1])(*tup[2])
-        except Exception:
-            return
-    elif code == EXT_CONSTRUCTOR_KW_ARGS:
-        try:
-            tup = ormsgpack.unpackb(
-                data, ext_hook=_msgpack_ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
-            )
-            # module, name, args
-            return getattr(importlib.import_module(tup[0]), tup[1])(**tup[2])
-        except Exception:
-            return
-    elif code == EXT_METHOD_SINGLE_ARG:
-        try:
-            tup = ormsgpack.unpackb(
-                data, ext_hook=_msgpack_ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
-            )
-            # module, name, arg, method
-            return getattr(getattr(importlib.import_module(tup[0]), tup[1]), tup[3])(
-                tup[2]
-            )
-        except Exception:
-            return
-    elif code == EXT_PYDANTIC_V1:
-        try:
-            tup = ormsgpack.unpackb(
-                data, ext_hook=_msgpack_ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
-            )
-            # module, name, kwargs
-            cls = getattr(importlib.import_module(tup[0]), tup[1])
-            try:
-                return cls(**tup[2])
-            except Exception:
-                return cls.construct(**tup[2])
-        except Exception:
-            # for pydantic objects we can't find/reconstruct
-            # let's return the kwargs dict instead
-            try:
-                return tup[2]
-            except NameError:
-                return
-    elif code == EXT_PYDANTIC_V2:
-        try:
-            tup = ormsgpack.unpackb(
-                data, ext_hook=_msgpack_ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
-            )
-            # module, name, kwargs, method
-            cls = getattr(importlib.import_module(tup[0]), tup[1])
-            try:
-                return cls(**tup[2])
-            except Exception:
-                return cls.model_construct(**tup[2])
-        except Exception:
-            # for pydantic objects we can't find/reconstruct
-            # let's return the kwargs dict instead
-            try:
-                return tup[2]
-            except NameError:
-                return
-    elif code == EXT_NUMPY_ARRAY:
-        try:
-            import numpy as _np
+def _create_msgpack_ext_hook(
+    allowed_modules: set[tuple[str, ...]] | Literal[True] | None,
+) -> Callable[[int, bytes], Any]:
+    """Create msgpack ext hook with allowlist.
 
-            dtype_str, shape, order, buf = ormsgpack.unpackb(
-                data, ext_hook=_msgpack_ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
+    Args:
+        allowed_modules: Set of (module, name) tuples that are allowed to be
+        deserialized, or True to allow all with warnings for unregistered types, or None to only allow safe types.
+
+    Returns:
+        An ext_hook function for use with ormsgpack.unpackb.
+    """
+
+    def _check_allowed(module: str, name: str) -> bool:
+        """Check if type is allowed. Returns True if allowed, False if blocked."""
+        key = (module, name)
+
+        if key in SAFE_MSGPACK_TYPES:
+            return True
+
+        if allowed_modules is not None and allowed_modules is not True:
+            if key in allowed_modules:
+                return True
+
+        if allowed_modules is True:
+            # default is to warn but allow unregistered types
+            logger.warning(
+                "Deserializing unregistered type %s.%s from checkpoint. "
+                "This will be blocked in a future version. "
+                "Add to allowed_msgpack_modules to silence: [(%r, %r)]",
+                module,
+                name,
+                module,
+                name,
             )
-            arr = _np.frombuffer(buf, dtype=_np.dtype(dtype_str))
-            return arr.reshape(shape, order=order)
-        except Exception:
-            return
+            return True
+        else:
+            # strict mode blocks unregistered types
+            logger.warning(
+                "Blocked deserialization of %s.%s - not in allowed_msgpack_modules. "
+                "Add to allowed_msgpack_modules to allow: [(%r, %r)]",
+                module,
+                name,
+                module,
+                name,
+            )
+            return False
+
+    def ext_hook(code: int, data: bytes) -> Any:
+        if code == EXT_CONSTRUCTOR_SINGLE_ARG:
+            try:
+                tup = ormsgpack.unpackb(
+                    data, ext_hook=ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
+                )
+                if not _check_allowed(tup[0], tup[1]):
+                    return None
+                # module, name, arg
+                return getattr(importlib.import_module(tup[0]), tup[1])(tup[2])
+            except Exception:
+                return None
+        elif code == EXT_CONSTRUCTOR_POS_ARGS:
+            try:
+                tup = ormsgpack.unpackb(
+                    data, ext_hook=ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
+                )
+                if not _check_allowed(tup[0], tup[1]):
+                    return None
+                # module, name, args
+                return getattr(importlib.import_module(tup[0]), tup[1])(*tup[2])
+            except Exception:
+                return None
+        elif code == EXT_CONSTRUCTOR_KW_ARGS:
+            try:
+                tup = ormsgpack.unpackb(
+                    data, ext_hook=ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
+                )
+                if not _check_allowed(tup[0], tup[1]):
+                    return None
+                # module, name, kwargs
+                return getattr(importlib.import_module(tup[0]), tup[1])(**tup[2])
+            except Exception:
+                return None
+        elif code == EXT_METHOD_SINGLE_ARG:
+            try:
+                tup = ormsgpack.unpackb(
+                    data, ext_hook=ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
+                )
+                if not _check_allowed(tup[0], tup[1]):
+                    return None
+                # module, name, arg, method
+                return getattr(
+                    getattr(importlib.import_module(tup[0]), tup[1]), tup[3]
+                )(tup[2])
+            except Exception:
+                return None
+        elif code == EXT_PYDANTIC_V1:
+            try:
+                tup = ormsgpack.unpackb(
+                    data, ext_hook=ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
+                )
+                if not _check_allowed(tup[0], tup[1]):
+                    return None
+                # module, name, kwargs
+                cls = getattr(importlib.import_module(tup[0]), tup[1])
+                try:
+                    return cls(**tup[2])
+                except Exception:
+                    return cls.construct(**tup[2])
+            except Exception:
+                # for pydantic objects we can't find/reconstruct
+                # let's return the kwargs dict instead
+                try:
+                    return tup[2]
+                except NameError:
+                    return None
+        elif code == EXT_PYDANTIC_V2:
+            try:
+                tup = ormsgpack.unpackb(
+                    data, ext_hook=ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
+                )
+                if not _check_allowed(tup[0], tup[1]):
+                    return None
+                # module, name, kwargs, method
+                cls = getattr(importlib.import_module(tup[0]), tup[1])
+                try:
+                    return cls(**tup[2])
+                except Exception:
+                    return cls.model_construct(**tup[2])
+            except Exception:
+                # for pydantic objects we can't find/reconstruct
+                # let's return the kwargs dict instead
+                try:
+                    return tup[2]
+                except NameError:
+                    return None
+        elif code == EXT_NUMPY_ARRAY:
+            try:
+                import numpy as _np
+
+                dtype_str, shape, order, buf = ormsgpack.unpackb(
+                    data, ext_hook=ext_hook, option=ormsgpack.OPT_NON_STR_KEYS
+                )
+                arr = _np.frombuffer(buf, dtype=_np.dtype(dtype_str))
+                return arr.reshape(shape, order=order)
+            except Exception:
+                return None
+        return None
+
+    return ext_hook
+
+
+_msgpack_ext_hook = _create_msgpack_ext_hook(allowed_modules=None)
 
 
 def _msgpack_ext_hook_to_json(code: int, data: bytes) -> Any:

libs/checkpoint/tests/test_jsonplus.py

@@ -37,6 +37,10 @@ class MyPydantic(BaseModel):
     inner: InnerPydantic
 
 
+class AnotherPydantic(BaseModel):
+    foo: str
+
+
 class InnerPydanticV1(BaseModelV1):
     hello: str
 
@@ -512,5 +516,240 @@ def test_serde_jsonplus_pandas_series(series: pd.Series) -> None:
 
     assert dumped[0] == "pickle"
     result = serde.loads_typed(dumped)
-
     assert result.equals(series)
+
+
+def test_msgpack_safe_types_no_warning(caplog: pytest.LogCaptureFixture) -> None:
+    """Test safe types deserialize without warnings."""
+
+    serde = JsonPlusSerializer()
+
+    safe_objects = [
+        datetime.now(),
+        date.today(),
+        time(12, 30),
+        timezone.utc,
+        uuid.uuid4(),
+        Decimal("123.45"),
+        {1, 2, 3},
+        frozenset([1, 2, 3]),
+        deque([1, 2, 3]),
+        IPv4Address("192.168.1.1"),
+        pathlib.Path("/tmp/test"),
+    ]
+
+    for obj in safe_objects:
+        caplog.clear()
+        dumped = serde.dumps_typed(obj)
+        result = serde.loads_typed(dumped)
+        assert "unregistered type" not in caplog.text.lower(), (
+            f"Unexpected warning for {type(obj)}"
+        )
+        assert result is not None
+
+
+def test_msgpack_pydantic_warns_by_default(caplog: pytest.LogCaptureFixture) -> None:
+    """Pydantic models not in allowlist should log warning but still deserialize.
+
+    TODO: We'll want to change this to block unregistered types in the future."""
+
+    serde = JsonPlusSerializer()
+
+    obj = MyPydantic(foo="test", bar=42, inner=InnerPydantic(hello="world"))
+
+    caplog.clear()
+    dumped = serde.dumps_typed(obj)
+    result = serde.loads_typed(dumped)
+
+    assert "unregistered type" in caplog.text.lower()
+    assert "allowed_msgpack_modules" in caplog.text
+    assert result == obj
+
+
+def test_msgpack_allowlist_silences_warning(caplog: pytest.LogCaptureFixture) -> None:
+    """Types in allowed_msgpack_modules should deserialize without warnings."""
+
+    serde = JsonPlusSerializer(
+        allowed_msgpack_modules=[
+            ("tests.test_jsonplus", "MyPydantic"),
+            ("tests.test_jsonplus", "InnerPydantic"),
+        ]
+    )
+
+    obj = MyPydantic(foo="test", bar=42, inner=InnerPydantic(hello="world"))
+
+    caplog.clear()
+    dumped = serde.dumps_typed(obj)
+    result = serde.loads_typed(dumped)
+
+    assert "unregistered type" not in caplog.text.lower()
+    assert result == obj
+
+
+def test_msgpack_none_blocks_unregistered(caplog: pytest.LogCaptureFixture) -> None:
+    """allowed_msgpack_modules=None should block unregistered types.
+
+    TODO: This will be the default behavior in the future."""
+
+    serde = JsonPlusSerializer(allowed_msgpack_modules=None)
+
+    obj = MyPydantic(foo="test", bar=42, inner=InnerPydantic(hello="world"))
+
+    caplog.clear()
+    dumped = serde.dumps_typed(obj)
+    result = serde.loads_typed(dumped)
+
+    assert "blocked" in caplog.text.lower()
+    assert result is None
+
+
+def test_msgpack_allowlist_blocks_non_listed(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Allowlists should block unregistered types even if msgpack is enabled."""
+
+    serde = JsonPlusSerializer(
+        allowed_msgpack_modules=[("tests.test_jsonplus", "MyPydantic")]
+    )
+
+    obj = AnotherPydantic(foo="nope")
+
+    caplog.clear()
+    dumped = serde.dumps_typed(obj)
+    result = serde.loads_typed(dumped)
+
+    assert "blocked" in caplog.text.lower()
+    assert result is None
+
+
+def test_msgpack_strict_allows_safe_types(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Safe types should still deserialize in strict mode without warnings."""
+
+    serde = JsonPlusSerializer(allowed_msgpack_modules=None)
+    safe = uuid.uuid4()
+
+    caplog.clear()
+    dumped = serde.dumps_typed(safe)
+    result = serde.loads_typed(dumped)
+
+    assert "blocked" not in caplog.text.lower()
+    assert result == safe
+
+
+def test_msgpack_regex_safe_type(caplog: pytest.LogCaptureFixture) -> None:
+    """re.compile patterns should deserialize without warnings as a safe type."""
+
+    serde = JsonPlusSerializer(allowed_msgpack_modules=None)
+    pattern = re.compile(r"foo.*bar", re.IGNORECASE | re.DOTALL)
+
+    caplog.clear()
+    dumped = serde.dumps_typed(pattern)
+    result = serde.loads_typed(dumped)
+
+    assert "blocked" not in caplog.text.lower()
+    assert "unregistered" not in caplog.text.lower()
+    assert result.pattern == pattern.pattern
+    assert result.flags == pattern.flags
+
+
+@pytest.mark.skipif(sys.version_info >= (3, 14), reason="pydantic v1 not on 3.14+")
+def test_msgpack_pydantic_v1_allowlist(caplog: pytest.LogCaptureFixture) -> None:
+    """Pydantic v1 models in allowlist should deserialize without warnings."""
+
+    serde = JsonPlusSerializer(
+        allowed_msgpack_modules=[
+            ("tests.test_jsonplus", "MyPydanticV1"),
+            ("tests.test_jsonplus", "InnerPydanticV1"),
+        ]
+    )
+
+    obj = MyPydanticV1(foo="test", bar=42, inner=InnerPydanticV1(hello="world"))
+
+    caplog.clear()
+    dumped = serde.dumps_typed(obj)
+    result = serde.loads_typed(dumped)
+
+    assert "unregistered type" not in caplog.text.lower()
+    assert "blocked" not in caplog.text.lower()
+    assert result == obj
+
+
+def test_msgpack_dataclass_allowlist(caplog: pytest.LogCaptureFixture) -> None:
+    """Dataclasses in allowlist should deserialize without warnings."""
+
+    serde = JsonPlusSerializer(
+        allowed_msgpack_modules=[
+            ("tests.test_jsonplus", "MyDataclass"),
+            ("tests.test_jsonplus", "InnerDataclass"),
+        ]
+    )
+
+    obj = MyDataclass(foo="test", bar=42, inner=InnerDataclass(hello="world"))
+
+    caplog.clear()
+    dumped = serde.dumps_typed(obj)
+    result = serde.loads_typed(dumped)
+
+    assert "unregistered type" not in caplog.text.lower()
+    assert "blocked" not in caplog.text.lower()
+    assert result == obj
+
+
+def test_msgpack_safe_types_value_equality(caplog: pytest.LogCaptureFixture) -> None:
+    """Verify safe types are correctly restored with proper values."""
+
+    serde = JsonPlusSerializer(allowed_msgpack_modules=None)
+
+    test_cases = [
+        datetime(2024, 1, 15, 12, 30, 45, 123456),
+        date(2024, 6, 15),
+        time(14, 30, 0),
+        uuid.UUID("12345678-1234-5678-1234-567812345678"),
+        Decimal("123.456789"),
+        {1, 2, 3, 4, 5},
+        frozenset(["a", "b", "c"]),
+        deque([1, 2, 3]),
+        IPv4Address("10.0.0.1"),
+        pathlib.Path("/some/test/path"),
+        re.compile(r"\d+", re.MULTILINE),
+    ]
+
+    for obj in test_cases:
+        caplog.clear()
+        dumped = serde.dumps_typed(obj)
+        result = serde.loads_typed(dumped)
+
+        assert "blocked" not in caplog.text.lower(), f"Blocked for {type(obj)}"
+        # For regex patterns, compare pattern and flags
+        if isinstance(obj, re.Pattern):
+            assert result.pattern == obj.pattern
+            assert result.flags == obj.flags
+        else:
+            assert result == obj, f"Value mismatch for {type(obj)}: {result} != {obj}"
+
+
+def test_msgpack_nested_pydantic_serializes_as_dict(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Nested Pydantic models are serialized via model_dump() as dicts.
+
+    This means nested models don't go through the ext hook and don't need
+    to be in the allowlist - only the outer type does.
+    """
+
+    # Only allow outer type - inner is serialized as dict via model_dump()
+    serde = JsonPlusSerializer(
+        allowed_msgpack_modules=[("tests.test_jsonplus", "MyPydantic")]
+    )
+
+    obj = MyPydantic(foo="test", bar=42, inner=InnerPydantic(hello="world"))
+
+    caplog.clear()
+    dumped = serde.dumps_typed(obj)
+    result = serde.loads_typed(dumped)
+
+    # No blocking should occur - inner is serialized as dict, not ext
+    assert "blocked" not in caplog.text.lower()
+    assert result == obj

libs/checkpoint/tests/test_memory.py

@@ -1,7 +1,9 @@
+import logging
 from typing import Any
 
 import pytest
 from langchain_core.runnables import RunnableConfig
+from pydantic import BaseModel
 
 from langgraph.checkpoint.base import (
     Checkpoint,
@@ -10,6 +12,11 @@ from langgraph.checkpoint.base import (
     empty_checkpoint,
 )
 from langgraph.checkpoint.memory import InMemorySaver
+from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
+
+
+class MemoryPydantic(BaseModel):
+    foo: str
 
 
 class TestMemorySaver:
@@ -199,3 +206,77 @@ async def test_memory_saver() -> None:
 
     with memory_saver as sync_memory_saver:
         assert sync_memory_saver is memory_saver
+
+
+def test_memory_saver_warns_on_unregistered_msgpack(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    serde = JsonPlusSerializer()
+    memory_saver = InMemorySaver(serde=serde)
+    obj = MemoryPydantic(foo="bar")
+
+    checkpoint = empty_checkpoint()
+    checkpoint["channel_values"] = {"foo": obj}
+    checkpoint["channel_versions"] = {"foo": 1}
+
+    config: RunnableConfig = {
+        "configurable": {"thread_id": "thread-1", "checkpoint_ns": ""}
+    }
+
+    caplog.set_level(logging.WARNING, logger="langgraph.checkpoint.serde.jsonplus")
+    new_config = memory_saver.put(config, checkpoint, {}, {"foo": 1})
+    result = memory_saver.get_tuple(new_config)
+
+    assert result is not None
+    assert "unregistered type" in caplog.text.lower()
+    assert result.checkpoint["channel_values"]["foo"] == obj
+
+
+def test_memory_saver_allowlist_silences_warning(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    serde = JsonPlusSerializer(
+        allowed_msgpack_modules=[("tests.test_memory", "MemoryPydantic")]
+    )
+    memory_saver = InMemorySaver(serde=serde)
+    obj = MemoryPydantic(foo="bar")
+
+    checkpoint = empty_checkpoint()
+    checkpoint["channel_values"] = {"foo": obj}
+    checkpoint["channel_versions"] = {"foo": 1}
+
+    config: RunnableConfig = {
+        "configurable": {"thread_id": "thread-1", "checkpoint_ns": ""}
+    }
+
+    caplog.set_level(logging.WARNING, logger="langgraph.checkpoint.serde.jsonplus")
+    new_config = memory_saver.put(config, checkpoint, {}, {"foo": 1})
+    result = memory_saver.get_tuple(new_config)
+
+    assert result is not None
+    assert "unregistered type" not in caplog.text.lower()
+    assert result.checkpoint["channel_values"]["foo"] == obj
+
+
+def test_memory_saver_strict_blocks_unregistered(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    serde = JsonPlusSerializer(allowed_msgpack_modules=None)
+    memory_saver = InMemorySaver(serde=serde)
+    obj = MemoryPydantic(foo="bar")
+
+    checkpoint = empty_checkpoint()
+    checkpoint["channel_values"] = {"foo": obj}
+    checkpoint["channel_versions"] = {"foo": 1}
+
+    config: RunnableConfig = {
+        "configurable": {"thread_id": "thread-1", "checkpoint_ns": ""}
+    }
+
+    caplog.set_level(logging.WARNING, logger="langgraph.checkpoint.serde.jsonplus")
+    new_config = memory_saver.put(config, checkpoint, {}, {"foo": 1})
+    result = memory_saver.get_tuple(new_config)
+
+    assert result is not None
+    assert "blocked" in caplog.text.lower()
+    assert result.checkpoint["channel_values"]["foo"] is None

libs/cli/langgraph_cli/schemas.py

@@ -126,7 +126,7 @@ class SerdeConfig(TypedDict, total=False):
     If omitted, no serde is set up (the object store will still be present, however)."""
 
     allowed_json_modules: list[list[str]] | bool | None
-    """Optional. List of allowed python modules to de-serialize custom objects from.
+    """Optional. List of allowed python modules to de-serialize custom objects from JSON.
     
     If provided, only the specified modules will be allowed to be deserialized.
     If omitted, no modules are allowed, and the object returned will simply be a json object OR
@@ -146,7 +146,34 @@ class SerdeConfig(TypedDict, total=False):
     Example:
     {...
         "serde": {
-            "allowed_json_modules": true
+            "allowed_json_modules": True
+        }
+    }
+
+    """
+    allowed_msgpack_modules: list[list[str]] | bool | None
+    """Optional. List of allowed python modules to de-serialize custom objects from msgpack.
+
+    Known safe types (langgraph.checkpoint.serde.jsonplus.SAFE_MSGPACK_TYPES) are always
+    allowed regardless of this setting. Use this to allowlist your custom Pydantic models,
+    dataclasses, and other user-defined types.
+
+    If True (default), unregistered types will log a warning but still be deserialized.
+    If None, only known safe types will be deserialized; unregistered types will be blocked.
+
+    Example - allowlist specific types (no warnings for these):
+    {...
+        "serde": {
+            "allowed_msgpack_modules": [
+                ["my_agent.models", "MyState"],
+            ]
+        }
+    }
+
+    Example - strict mode (only safe types allowed):
+    {...
+        "serde": {
+            "allowed_msgpack_modules": null
         }
     }
     
@@ -306,8 +333,7 @@ class EncryptionConfig(TypedDict, total=False):
     """Configuration for custom at-rest encryption logic.
 
     Allows you to implement custom encryption for sensitive data stored in the database,
-    including metadata fields and checkpoint blobs.
-    """
+    including metadata fields and checkpoint blobs."""
 
     path: str
     """Required. Path to an instance of the Encryption() class that implements custom encryption handlers.

libs/cli/schemas/schema.json

@@ -591,7 +591,27 @@
               "type": "null"
             }
           ],
-          "description": "Optional. List of allowed python modules to de-serialize custom objects from.\n\nIf provided, only the specified modules will be allowed to be deserialized.\nIf omitted, no modules are allowed, and the object returned will simply be a json object OR\na deserialized langchain object.\n"
+          "description": "Optional. List of allowed python modules to de-serialize custom objects from JSON.\n\nIf provided, only the specified modules will be allowed to be deserialized.\nIf omitted, no modules are allowed, and the object returned will simply be a json object OR\na deserialized langchain object.\n"
+        },
+        "allowed_msgpack_modules": {
+          "anyOf": [
+            {
+              "type": "array",
+              "items": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              }
+            },
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "description": "Optional. List of allowed python modules to de-serialize custom objects from msgpack.\n\nKnown safe types (langgraph.checkpoint.serde.jsonplus.SAFE_MSGPACK_TYPES) are always\nallowed regardless of this setting. Use this to allowlist your custom Pydantic models,\ndataclasses, and other user-defined types.\n\nIf True (default), unregistered types will log a warning but still be deserialized.\nIf None, only known safe types will be deserialized; unregistered types will be blocked.\n\n{...\n[\"my_agent.models\", \"MyState\"],\n]\n}\n}\n\n{...\n}\n}\n\n"
         },
         "pickle_fallback": {
           "type": "boolean",

libs/cli/schemas/schema.v0.json

@@ -591,7 +591,27 @@
               "type": "null"
             }
           ],
-          "description": "Optional. List of allowed python modules to de-serialize custom objects from.\n\nIf provided, only the specified modules will be allowed to be deserialized.\nIf omitted, no modules are allowed, and the object returned will simply be a json object OR\na deserialized langchain object.\n"
+          "description": "Optional. List of allowed python modules to de-serialize custom objects from JSON.\n\nIf provided, only the specified modules will be allowed to be deserialized.\nIf omitted, no modules are allowed, and the object returned will simply be a json object OR\na deserialized langchain object.\n"
+        },
+        "allowed_msgpack_modules": {
+          "anyOf": [
+            {
+              "type": "array",
+              "items": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              }
+            },
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "description": "Optional. List of allowed python modules to de-serialize custom objects from msgpack.\n\nKnown safe types (langgraph.checkpoint.serde.jsonplus.SAFE_MSGPACK_TYPES) are always\nallowed regardless of this setting. Use this to allowlist your custom Pydantic models,\ndataclasses, and other user-defined types.\n\nIf True (default), unregistered types will log a warning but still be deserialized.\nIf None, only known safe types will be deserialized; unregistered types will be blocked.\n\n{...\n[\"my_agent.models\", \"MyState\"],\n]\n}\n}\n\n{...\n}\n}\n\n"
         },
         "pickle_fallback": {
           "type": "boolean",

libs/langgraph/tests/conftest_checkpointer.py

@@ -1,3 +1,4 @@
+import os
 from contextlib import asynccontextmanager, contextmanager
 from uuid import uuid4
 
@@ -5,6 +6,7 @@ import pytest
 from langgraph.checkpoint.postgres import PostgresSaver
 from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
 from langgraph.checkpoint.serde.encrypted import EncryptedSerializer
+from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
 from langgraph.checkpoint.sqlite import SqliteSaver
 from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
 from psycopg import AsyncConnection, Connection
@@ -18,30 +20,60 @@ from tests.memory_assert import (  # noqa: E402
 )
 
 DEFAULT_POSTGRES_URI = "postgres://postgres:postgres@localhost:5442/"
+STRICT_MSGPACK = os.getenv("LANGGRAPH_STRICT_MSGPACK", "false").lower() in (
+    "1",
+    "true",
+    "yes",
+)
+
+
+def _strict_msgpack_serde() -> JsonPlusSerializer:
+    return JsonPlusSerializer(allowed_msgpack_modules=None)
+
+
+def _apply_strict_msgpack(checkpointer) -> None:
+    if not STRICT_MSGPACK:
+        return
+    serde = _strict_msgpack_serde()
+    if hasattr(checkpointer, "serde"):
+        checkpointer.serde = serde
+    if hasattr(checkpointer, "saver") and hasattr(checkpointer.saver, "serde"):
+        checkpointer.saver.serde = serde
 
 
 @contextmanager
 def _checkpointer_memory():
-    yield MemorySaverAssertImmutable()
+    if STRICT_MSGPACK:
+        yield MemorySaverAssertImmutable(serde=_strict_msgpack_serde())
+    else:
+        yield MemorySaverAssertImmutable()
 
 
 @contextmanager
 def _checkpointer_memory_migrate_sends():
-    yield MemorySaverNeedsPendingSendsMigration()
+    checkpointer = MemorySaverNeedsPendingSendsMigration()
+    _apply_strict_msgpack(checkpointer)
+    yield checkpointer
 
 
 @contextmanager
 def _checkpointer_sqlite():
     with SqliteSaver.from_conn_string(":memory:") as checkpointer:
+        _apply_strict_msgpack(checkpointer)
         yield checkpointer
 
 
 @contextmanager
 def _checkpointer_sqlite_aes():
     with SqliteSaver.from_conn_string(":memory:") as checkpointer:
-        checkpointer.serde = EncryptedSerializer.from_pycryptodome_aes(
-            key=b"1234567890123456"
-        )
+        if STRICT_MSGPACK:
+            checkpointer.serde = EncryptedSerializer.from_pycryptodome_aes(
+                serde=_strict_msgpack_serde(), key=b"1234567890123456"
+            )
+        else:
+            checkpointer.serde = EncryptedSerializer.from_pycryptodome_aes(
+                key=b"1234567890123456"
+            )
         yield checkpointer
 
 
@@ -57,6 +89,7 @@ def _checkpointer_postgres():
             DEFAULT_POSTGRES_URI + database
         ) as checkpointer:
             checkpointer.setup()
+            _apply_strict_msgpack(checkpointer)
             yield checkpointer
     finally:
         # drop unique db
@@ -79,6 +112,7 @@ def _checkpointer_postgres_pipe():
             # setup can't run inside pipeline because of implicit transaction
             with checkpointer.conn.pipeline() as pipe:
                 checkpointer.pipe = pipe
+                _apply_strict_msgpack(checkpointer)
                 yield checkpointer
     finally:
         # drop unique db
@@ -99,6 +133,7 @@ def _checkpointer_postgres_pool():
         ) as pool:
             checkpointer = PostgresSaver(pool)
             checkpointer.setup()
+            _apply_strict_msgpack(checkpointer)
             yield checkpointer
     finally:
         # drop unique db
@@ -109,6 +144,7 @@ def _checkpointer_postgres_pool():
 @asynccontextmanager
 async def _checkpointer_sqlite_aio():
     async with AsyncSqliteSaver.from_conn_string(":memory:") as checkpointer:
+        _apply_strict_msgpack(checkpointer)
         yield checkpointer
 
 
@@ -126,6 +162,7 @@ async def _checkpointer_postgres_aio():
             DEFAULT_POSTGRES_URI + database
         ) as checkpointer:
             await checkpointer.setup()
+            _apply_strict_msgpack(checkpointer)
             yield checkpointer
     finally:
         # drop unique db
@@ -152,6 +189,7 @@ async def _checkpointer_postgres_aio_pipe():
             # setup can't run inside pipeline because of implicit transaction
             async with checkpointer.conn.pipeline() as pipe:
                 checkpointer.pipe = pipe
+                _apply_strict_msgpack(checkpointer)
                 yield checkpointer
     finally:
         # drop unique db
@@ -176,6 +214,7 @@ async def _checkpointer_postgres_aio_pool():
         ) as pool:
             checkpointer = AsyncPostgresSaver(pool)
             await checkpointer.setup()
+            _apply_strict_msgpack(checkpointer)
             yield checkpointer
     finally:
         # drop unique db

libs/sdk-py/langgraph_sdk/__init__.py

@@ -3,6 +3,6 @@ from langgraph_sdk.client import get_client, get_sync_client
 from langgraph_sdk.encryption import Encryption
 from langgraph_sdk.encryption.types import EncryptionContext
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"
 
 __all__ = ["Auth", "Encryption", "EncryptionContext", "get_client", "get_sync_client"]

libs/sdk-py/langgraph_sdk/errors.py

@@ -38,7 +38,7 @@ class APIError(httpx.HTTPStatusError, LangGraphError):
             response = None
 
         httpx.HTTPStatusError.__init__(self, message, request=req, response=response)  # type: ignore[arg-type]
-        LangGraphError.__init__(self)
+        LangGraphError.__init__(self, message)
 
         self.request = req
         self.message = message

libs/sdk-py/tests/test_errors.py

@@ -106,3 +106,39 @@ def test_field_extraction_from_json_body() -> None:
     assert err.code == "invalid_param"
     assert err.param == "limit"
     assert err.type == "invalid_request_error"
+
+
+def test_error_message_in_str_and_args() -> None:
+    """Test that error message is accessible via str() and args."""
+    r = make_response(422, json_body={"message": "Validation failed"})
+    with pytest.raises(UnprocessableEntityError) as ei:
+        _raise_for_status_typed(r)
+    err = cast("UnprocessableEntityError", ei.value)
+    assert str(err) == "Validation failed"
+    assert err.args == ("Validation failed",)
+    assert err.message == "Validation failed"
+
+
+@pytest.mark.parametrize(
+    "status,exc_type",
+    [
+        (400, BadRequestError),
+        (401, AuthenticationError),
+        (403, PermissionDeniedError),
+        (404, NotFoundError),
+        (409, ConflictError),
+        (422, UnprocessableEntityError),
+        (429, RateLimitError),
+        (500, InternalServerError),
+    ],
+)
+def test_all_error_types_display_message(
+    status: int, exc_type: type[APIStatusError]
+) -> None:
+    """Test that all error subclasses properly display their message."""
+    r = make_response(status, json_body={"message": "test error message"})
+    with pytest.raises(exc_type) as ei:
+        _raise_for_status_typed(r)
+    err = ei.value
+    assert str(err) == "test error message"
+    assert "test error message" in err.args