[Docs] Use injected RunnableConfig (#1444)

* Pass via type

* Format
This commit is contained in:
William FH
2024-08-22 18:54:42 -07:00
committed by GitHub
parent 6ece7124ed
commit dec7eb6f58
19 changed files with 127 additions and 79 deletions
@@ -235,7 +235,7 @@
" # Call the chat bot\n",
" chat_bot_response = my_chat_bot(messages)\n",
" # Respond with an AI Message\n",
" return {\"messages\":[AIMessage(content=chat_bot_response[\"content\"])]}"
" return {\"messages\": [AIMessage(content=chat_bot_response[\"content\"])]}"
]
},
{
@@ -270,7 +270,7 @@
" # Call the simulated user\n",
" response = simulated_user.invoke({\"messages\": new_messages})\n",
" # This response is an AI message - we need to flip this to be a human message\n",
" return {\"messages\":[HumanMessage(content=response.content)]}"
" return {\"messages\": [HumanMessage(content=response.content)]}"
]
},
{
@@ -331,6 +331,7 @@
"class State(TypedDict):\n",
" messages: Annotated[list, add_messages]\n",
"\n",
"\n",
"graph_builder = StateGraph(State)\n",
"graph_builder.add_node(\"user\", simulated_user_node)\n",
"graph_builder.add_node(\"chat_bot\", chat_bot_node)\n",
@@ -79,7 +79,7 @@
"\n",
"\n",
"def info_chain(state):\n",
" messages = get_messages_info(state['messages'])\n",
" messages = get_messages_info(state[\"messages\"])\n",
" response = llm_with_tool.invoke(messages)\n",
" return {\"messages\": [response]}"
]
@@ -126,7 +126,7 @@
"\n",
"\n",
"def prompt_gen_chain(state):\n",
" messages = get_prompt_messages(state['messages'])\n",
" messages = get_prompt_messages(state[\"messages\"])\n",
" response = llm.invoke(messages)\n",
" return {\"messages\": [response]}"
]
@@ -158,7 +158,7 @@
"\n",
"\n",
"def get_state(state) -> Literal[\"add_tool_message\", \"info\", \"__end__\"]:\n",
" messages = state['messages']\n",
" messages = state[\"messages\"]\n",
" if isinstance(messages[-1], AIMessage) and messages[-1].tool_calls:\n",
" return \"add_tool_message\"\n",
" elif not isinstance(messages[-1], HumanMessage):\n",
@@ -190,9 +190,11 @@
"from typing import Annotated\n",
"from typing_extensions import TypedDict\n",
"\n",
"\n",
"class State(TypedDict):\n",
" messages: Annotated[list, add_messages]\n",
"\n",
"\n",
"memory = MemorySaver()\n",
"workflow = StateGraph(State)\n",
"workflow.add_node(\"info\", info_chain)\n",
@@ -201,9 +203,14 @@
"\n",
"@workflow.add_node\n",
"def add_tool_message(state: State):\n",
" return {\"messages\": [ToolMessage(\n",
" content=\"Prompt generated!\", tool_call_id=state['messages'][-1].tool_calls[0][\"id\"]\n",
" )]}\n",
" return {\n",
" \"messages\": [\n",
" ToolMessage(\n",
" content=\"Prompt generated!\",\n",
" tool_call_id=state[\"messages\"][-1].tool_calls[0][\"id\"],\n",
" )\n",
" ]\n",
" }\n",
"\n",
"\n",
"workflow.add_conditional_edges(\"info\", get_state)\n",
@@ -364,7 +371,7 @@
" for output in graph.stream(\n",
" {\"messages\": [HumanMessage(content=user)]}, config=config, stream_mode=\"updates\"\n",
" ):\n",
" last_message = next(iter(output.values()))['messages'][-1]\n",
" last_message = next(iter(output.values()))[\"messages\"][-1]\n",
" last_message.pretty_print()\n",
"\n",
" if output and \"prompt\" in output:\n",
+1 -1
View File
@@ -239,7 +239,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.1"
"version": "3.12.2"
}
},
"nbformat": 4,
@@ -225,7 +225,14 @@
"\n",
"Define the (`fetch_user_flight_information`) tool to let the agent see the current user's flight information. Then define tools to search for flights and manage the passenger's bookings stored in the SQL database.\n",
"\n",
"We use `ensure_config` to pass in the `passenger_id` in via configurable parameters. The LLM never has to provide these explicitly, they are provided for a given invocation of the graph so that each user cannot access other passengers' booking information."
"We the can [access the RunnableConfig](https://python.langchain.com/v0.2/docs/how_to/tool_configure/#inferring-by-parameter-type) for a given run to check the `passenger_id` of the user accessing this application. The LLM never has to provide these explicitly, they are provided for a given invocation of the graph so that each user cannot access other passengers' booking information.\n",
"\n",
"<div class=\"admonition warning\">\n",
" <p class=\"admonition-title\">Compatibility</p>\n",
" <p>\n",
" This tutorial expects `langchain-core>=0.2.16` to use the injected RunnableConfig. Prior to that, you'd use `ensure_config` to collect the config from context.\n",
" </p>\n",
"</div> \n"
]
},
{
@@ -240,18 +247,17 @@
"from typing import Optional\n",
"\n",
"import pytz\n",
"from langchain_core.runnables import ensure_config\n",
"from langchain_core.runnables import RunnableConfig\n",
"\n",
"\n",
"@tool\n",
"def fetch_user_flight_information() -> list[dict]:\n",
"def fetch_user_flight_information(config: RunnableConfig) -> list[dict]:\n",
" \"\"\"Fetch all tickets for the user along with corresponding flight information and seat assignments.\n",
"\n",
" Returns:\n",
" A list of dictionaries where each dictionary contains the ticket details,\n",
" associated flight details, and the seat assignments for each ticket belonging to the user.\n",
" \"\"\"\n",
" config = ensure_config() # Fetch from the context\n",
" configuration = config.get(\"configurable\", {})\n",
" passenger_id = configuration.get(\"passenger_id\", None)\n",
" if not passenger_id:\n",
@@ -328,9 +334,10 @@
"\n",
"\n",
"@tool\n",
"def update_ticket_to_new_flight(ticket_no: str, new_flight_id: int) -> str:\n",
"def update_ticket_to_new_flight(\n",
" ticket_no: str, new_flight_id: int, *, config: RunnableConfig\n",
") -> str:\n",
" \"\"\"Update the user's ticket to a new valid flight.\"\"\"\n",
" config = ensure_config()\n",
" configuration = config.get(\"configurable\", {})\n",
" passenger_id = configuration.get(\"passenger_id\", None)\n",
" if not passenger_id:\n",
@@ -396,9 +403,8 @@
"\n",
"\n",
"@tool\n",
"def cancel_ticket(ticket_no: str) -> str:\n",
"def cancel_ticket(ticket_no: str, *, config: RunnableConfig) -> str:\n",
" \"\"\"Cancel the user's ticket and remove it from the database.\"\"\"\n",
" config = ensure_config()\n",
" configuration = config.get(\"configurable\", {})\n",
" passenger_id = configuration.get(\"passenger_id\", None)\n",
" if not passenger_id:\n",
@@ -4407,7 +4413,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.2"
"version": "3.12.2"
}
},
"nbformat": 4,
@@ -141,15 +141,18 @@
" print(\"----\")\n",
" return \"Sunny!\"\n",
"\n",
"model = ChatAnthropic(model_name=\"claude-3-5-sonnet-20240620\").bind_tools([weather_search])\n",
"\n",
"model = ChatAnthropic(model_name=\"claude-3-5-sonnet-20240620\").bind_tools(\n",
" [weather_search]\n",
")\n",
"\n",
"\n",
"class State(MessagesState):\n",
" \"\"\"Simple state.\"\"\"\n",
"\n",
"\n",
"def call_llm(state):\n",
" return {\n",
" \"messages\": [model.invoke(state['messages'])]\n",
" }\n",
" return {\"messages\": [model.invoke(state[\"messages\"])]}\n",
"\n",
"\n",
"def human_review_node(state):\n",
@@ -159,28 +162,30 @@
"def run_tool(state):\n",
" new_messages = []\n",
" tools = {\"weather_search\": weather_search}\n",
" tool_calls = state['messages'][-1].tool_calls\n",
" tool_calls = state[\"messages\"][-1].tool_calls\n",
" for tool_call in tool_calls:\n",
" tool = tools[tool_call['name']]\n",
" result = tool.invoke(tool_call['args'])\n",
" new_messages.append({\n",
" \"role\": \"tool\",\n",
" \"name\": tool_call['name'],\n",
" \"content\": result,\n",
" \"tool_call_id\": tool_call['id']\n",
" })\n",
" tool = tools[tool_call[\"name\"]]\n",
" result = tool.invoke(tool_call[\"args\"])\n",
" new_messages.append(\n",
" {\n",
" \"role\": \"tool\",\n",
" \"name\": tool_call[\"name\"],\n",
" \"content\": result,\n",
" \"tool_call_id\": tool_call[\"id\"],\n",
" }\n",
" )\n",
" return {\"messages\": new_messages}\n",
"\n",
"\n",
"def route_after_llm(state) -> Literal[END, \"human_review_node\"]:\n",
" if len(state['messages'][-1].tool_calls) == 0:\n",
" if len(state[\"messages\"][-1].tool_calls) == 0:\n",
" return END\n",
" else:\n",
" return \"human_review_node\"\n",
"\n",
"\n",
"def route_after_human(state) -> Literal[\"run_tool\", \"call_llm\"]:\n",
" if isinstance(state['messages'][-1], AIMessage):\n",
" if isinstance(state[\"messages\"][-1], AIMessage):\n",
" return \"run_tool\"\n",
" else:\n",
" return \"call_llm\"\n",
@@ -460,35 +465,35 @@
"print(\"Current State:\")\n",
"print(state.values)\n",
"print(\"\\nCurrent Tool Call ID:\")\n",
"current_content = state.values['messages'][-1].content\n",
"current_id = state.values['messages'][-1].id\n",
"tool_call_id = state.values['messages'][-1].tool_calls[0]['id']\n",
"current_content = state.values[\"messages\"][-1].content\n",
"current_id = state.values[\"messages\"][-1].id\n",
"tool_call_id = state.values[\"messages\"][-1].tool_calls[0][\"id\"]\n",
"print(tool_call_id)\n",
"\n",
"# We now need to construct a replacement tool call.\n",
"# We will change the argument to be `San Francisco, USA`\n",
"# Note that we could change any number of arguments or tool names - it just has to be a valid one\n",
"new_message = {\n",
" \"role\": \"assistant\", \n",
" \"role\": \"assistant\",\n",
" \"content\": current_content,\n",
" \"tool_calls\": [\n",
" {\n",
" \"id\": tool_call_id,\n",
" \"name\": \"weather_search\",\n",
" \"args\": {\"city\": \"San Francisco, USA\"}\n",
" \"args\": {\"city\": \"San Francisco, USA\"},\n",
" }\n",
" ],\n",
" # This is important - this needs to be the same as the message you replacing!\n",
" # Otherwise, it will show up as a separate message\n",
" \"id\": current_id\n",
" \"id\": current_id,\n",
"}\n",
"graph.update_state(\n",
" # This is the config which represents this thread\n",
" thread, \n",
" thread,\n",
" # This is the updated value we want to push\n",
" {\"messages\": [new_message]}, \n",
" {\"messages\": [new_message]},\n",
" # We push this update acting as our human_review_node\n",
" as_node=\"human_review_node\"\n",
" as_node=\"human_review_node\",\n",
")\n",
"\n",
"# Let's now continue executing from here\n",
@@ -595,26 +600,26 @@
"print(\"Current State:\")\n",
"print(state.values)\n",
"print(\"\\nCurrent Tool Call ID:\")\n",
"tool_call_id = state.values['messages'][-1].tool_calls[0]['id']\n",
"tool_call_id = state.values[\"messages\"][-1].tool_calls[0][\"id\"]\n",
"print(tool_call_id)\n",
"\n",
"# We now need to construct a replacement tool call.\n",
"# We will change the argument to be `San Francisco, USA`\n",
"# Note that we could change any number of arguments or tool names - it just has to be a valid one\n",
"new_message = {\n",
" \"role\": \"tool\", \n",
" \"role\": \"tool\",\n",
" # This is our natural language feedback\n",
" \"content\": \"User requested changes: pass in the country as well\",\n",
" \"name\": \"weather_search\",\n",
" \"tool_call_id\": tool_call_id\n",
" \"tool_call_id\": tool_call_id,\n",
"}\n",
"graph.update_state(\n",
" # This is the config which represents this thread\n",
" thread, \n",
" thread,\n",
" # This is the updated value we want to push\n",
" {\"messages\": [new_message]}, \n",
" {\"messages\": [new_message]},\n",
" # We push this update acting as our human_review_node\n",
" as_node=\"human_review_node\"\n",
" as_node=\"human_review_node\",\n",
")\n",
"\n",
"# Let's now continue executing from here\n",
+4
View File
@@ -33,15 +33,19 @@
"from langgraph.graph import StateGraph, START, END\n",
"from typing import TypedDict\n",
"\n",
"\n",
"class InputState(TypedDict):\n",
" question: str\n",
"\n",
"\n",
"class OutputState(TypedDict):\n",
" answer: str\n",
"\n",
"\n",
"def answer_node(state: InputState):\n",
" return {\"answer\": \"bye\"}\n",
"\n",
"\n",
"graph = StateGraph(input=InputState, output=OutputState)\n",
"graph.add_node(answer_node)\n",
"graph.add_edge(START, \"answer_node\")\n",
+7 -3
View File
@@ -526,7 +526,7 @@
" \"tasks\": tasks,\n",
" }\n",
" )\n",
" return {\"messages\":[scheduled_tasks]}"
" return {\"messages\": [scheduled_tasks]}"
]
},
{
@@ -653,7 +653,7 @@
" )\n",
" ]\n",
" else:\n",
" return {\"messages\":response + [AIMessage(content=decision.action.response)]}\n",
" return {\"messages\": response + [AIMessage(content=decision.action.response)]}\n",
"\n",
"\n",
"def select_recent_messages(state) -> dict:\n",
@@ -726,9 +726,11 @@
"from langgraph.graph.message import add_messages\n",
"from typing import Annotated\n",
"\n",
"\n",
"class State(TypedDict):\n",
" messages: Annotated[list, add_messages]\n",
"\n",
"\n",
"graph_builder = StateGraph(State)\n",
"\n",
"# 1. Define vertices\n",
@@ -794,7 +796,9 @@
}
],
"source": [
"for step in chain.stream({\"messages\":[HumanMessage(content=\"What's the GDP of New York?\")]}):\n",
"for step in chain.stream(\n",
" {\"messages\": [HumanMessage(content=\"What's the GDP of New York?\")]}\n",
"):\n",
" print(step)\n",
" print(\"---\")"
]
+3 -3
View File
@@ -328,9 +328,9 @@
" \"set more_information_needed False and populate a blank string for the query.\"\n",
" )\n",
" input_messages = [system] + state[\"messages\"]\n",
" response = llm.bind_tools(\n",
" [QueryForTools], tool_choice=True\n",
" ).invoke(input_messages)\n",
" response = llm.bind_tools([QueryForTools], tool_choice=True).invoke(\n",
" input_messages\n",
" )\n",
" query = response.tool_calls[0][\"args\"][\"query\"]\n",
" tool_documents = vector_store.similarity_search(query)\n",
" if hack_remove_tool_condition:\n",
@@ -329,6 +329,7 @@
"\n",
"tools = [get_context, cite_context_sources]\n",
"\n",
"\n",
"# Define the function that calls the model\n",
"def call_model(state, config):\n",
" messages = state[\"messages\"]\n",
+2 -2
View File
@@ -72,12 +72,12 @@
"# Node to retrieve documents\n",
"def retrieve_documents(state: QueryOutputState) -> DocumentOutputState:\n",
" # Replace this with real logic\n",
" return {\"docs\": [state['query']] * 2}\n",
" return {\"docs\": [state[\"query\"]] * 2}\n",
"\n",
"\n",
"# Node to generate answer\n",
"def generate(state: GenerateInputState) -> OverallState:\n",
" return {\"answer\": \"\\n\\n\".join(state['docs'] + [state['question']])}\n",
" return {\"answer\": \"\\n\\n\".join(state[\"docs\"] + [state[\"question\"]])}\n",
"\n",
"\n",
"graph = StateGraph(OverallState)\n",
+1 -1
View File
@@ -587,7 +587,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.9"
"version": "3.12.2"
}
},
"nbformat": 4,
+10 -4
View File
@@ -630,7 +630,7 @@
" upsert=True,\n",
" )\n",
" )\n",
" await self.db[\"checkpoint_writes\"].bulk_write(operations)\n"
" await self.db[\"checkpoint_writes\"].bulk_write(operations)"
]
},
{
@@ -685,7 +685,9 @@
"metadata": {},
"outputs": [],
"source": [
"with MongoDBSaver.from_conn_info(host=\"localhost\", port=27017, db_name=\"checkpoints\") as checkpointer:\n",
"with MongoDBSaver.from_conn_info(\n",
" host=\"localhost\", port=27017, db_name=\"checkpoints\"\n",
") as checkpointer:\n",
" graph = create_react_agent(model, tools=tools, checkpointer=checkpointer)\n",
" config = {\"configurable\": {\"thread_id\": \"1\"}}\n",
" res = graph.invoke({\"messages\": [(\"human\", \"what's the weather in sf\")]}, config)\n",
@@ -796,10 +798,14 @@
"metadata": {},
"outputs": [],
"source": [
"async with AsyncMongoDBSaver.from_conn_info(host=\"localhost\", port=27017, db_name=\"checkpoints\") as checkpointer:\n",
"async with AsyncMongoDBSaver.from_conn_info(\n",
" host=\"localhost\", port=27017, db_name=\"checkpoints\"\n",
") as checkpointer:\n",
" graph = create_react_agent(model, tools=tools, checkpointer=checkpointer)\n",
" config = {\"configurable\": {\"thread_id\": \"2\"}}\n",
" res = await graph.ainvoke({\"messages\": [(\"human\", \"what's the weather in nyc\")]}, config)\n",
" res = await graph.ainvoke(\n",
" {\"messages\": [(\"human\", \"what's the weather in nyc\")]}, config\n",
" )\n",
"\n",
" latest_checkpoint = await checkpointer.aget(config)\n",
" latest_checkpoint_tuple = await checkpointer.aget_tuple(config)\n",
+3 -3
View File
@@ -134,7 +134,7 @@
"source": [
"from psycopg.rows import dict_row\n",
"\n",
"connection_kwargs ={\n",
"connection_kwargs = {\n",
" \"autocommit\": True,\n",
" \"prepare_threshold\": 0,\n",
"}"
@@ -165,7 +165,7 @@
" # Example configuration\n",
" conninfo=DB_URI,\n",
" max_size=20,\n",
" kwargs=connection_kwargs\n",
" kwargs=connection_kwargs,\n",
")\n",
"\n",
"with pool.connection() as conn:\n",
@@ -393,7 +393,7 @@
" # Example configuration\n",
" conninfo=DB_URI,\n",
" max_size=20,\n",
" kwargs=connection_kwargs\n",
" kwargs=connection_kwargs,\n",
") as pool, pool.connection() as conn:\n",
" checkpointer = AsyncPostgresSaver(conn)\n",
"\n",
+9 -3
View File
@@ -530,7 +530,9 @@
"\n",
" @classmethod\n",
" @asynccontextmanager\n",
" async def from_conn_info(cls, *, host: str, port: int, db: int) -> AsyncIterator[\"AsyncRedisSaver\"]:\n",
" async def from_conn_info(\n",
" cls, *, host: str, port: int, db: int\n",
" ) -> AsyncIterator[\"AsyncRedisSaver\"]:\n",
" conn = None\n",
" try:\n",
" conn = AsyncRedis(host=host, port=port, db=db)\n",
@@ -887,10 +889,14 @@
"metadata": {},
"outputs": [],
"source": [
"async with AsyncRedisSaver.from_conn_info(host=\"localhost\", port=6379, db=0) as checkpointer:\n",
"async with AsyncRedisSaver.from_conn_info(\n",
" host=\"localhost\", port=6379, db=0\n",
") as checkpointer:\n",
" graph = create_react_agent(model, tools=tools, checkpointer=checkpointer)\n",
" config = {\"configurable\": {\"thread_id\": \"2\"}}\n",
" res = await graph.ainvoke({\"messages\": [(\"human\", \"what's the weather in nyc\")]}, config)\n",
" res = await graph.ainvoke(\n",
" {\"messages\": [(\"human\", \"what's the weather in nyc\")]}, config\n",
" )\n",
"\n",
" latest_checkpoint = await checkpointer.aget(config)\n",
" latest_checkpoint_tuple = await checkpointer.aget_tuple(config)\n",
+1 -1
View File
@@ -269,7 +269,7 @@
"class State(TypedDict):\n",
" messages: Annotated[list, add_messages]\n",
"\n",
" \n",
"\n",
"async def generation_node(state: Sequence[BaseMessage]):\n",
" return await generate.ainvoke({\"messages\": state})\n",
"\n",
+1
View File
@@ -392,6 +392,7 @@
"class State(TypedDict):\n",
" messages: Annotated[list, add_messages]\n",
"\n",
"\n",
"MAX_ITERATIONS = 5\n",
"builder = StateGraph(State)\n",
"builder.add_node(\"draft\", first_responder.respond)\n",
+3 -1
View File
@@ -68,7 +68,9 @@
" # It's completely optional, but useful if you have many functions with similar names\n",
" gen = RunnableGenerator(my_generator).with_config(\n",
" tags=[\"should_stream\"],\n",
" callbacks=config.get(\"callbacks\", []) # <-- Propagate callbacks (Python <= 3.10)\n",
" callbacks=config.get(\n",
" \"callbacks\", []\n",
" ), # <-- Propagate callbacks (Python <= 3.10)\n",
" )\n",
" async for message in gen.astream(state):\n",
" messages.append(message)\n",
@@ -169,9 +169,7 @@
"from langchain_core.output_parsers import JsonOutputParser\n",
"\n",
"# JSON\n",
"llm = ChatOllama(model=\"llama3.1\", \n",
" format=\"json\", \n",
" temperature=0)\n",
"llm = ChatOllama(model=\"llama3.1\", format=\"json\", temperature=0)\n",
"\n",
"\n",
"prompt = PromptTemplate(\n",
@@ -210,6 +208,7 @@
"from IPython.display import Image, display\n",
"from langgraph.graph import START, END, StateGraph\n",
"\n",
"\n",
"class GraphState(TypedDict):\n",
" \"\"\"\n",
" Represents the state of our graph.\n",
@@ -381,21 +380,22 @@
"metadata": {},
"outputs": [],
"source": [
"import uuid \n",
"import uuid\n",
"\n",
"\n",
"def predict_custom_agent_answer(example: dict):\n",
" \n",
" config = {\"configurable\": {\"thread_id\": str(uuid.uuid4())}}\n",
" \n",
"\n",
" state_dict = custom_graph.invoke(\n",
" {\"question\": example[\"input\"], \"steps\": []}, config\n",
" )\n",
" \n",
"\n",
" return {\"response\": state_dict[\"generation\"], \"steps\": state_dict[\"steps\"]}\n",
"\n",
"\n",
"example = {\"input\": \"What are the types of agent memory?\"}\n",
"#response = predict_custom_agent_answer(example)\n",
"#response"
"# response = predict_custom_agent_answer(example)\n",
"# response"
]
},
{
@@ -544,6 +544,7 @@
" \"generate_answer\",\n",
"]\n",
"\n",
"\n",
"def check_trajectory_custom(root_run: Run, example: Example) -> dict:\n",
" \"\"\"\n",
" Check if all expected tools are called in exact order and without any additional tool calls.\n",
@@ -134,6 +134,7 @@
" for d in web_results\n",
" ]\n",
"\n",
"\n",
"# Tool list\n",
"tools = [retrieve_documents, web_search]"
]
@@ -152,9 +153,11 @@
"from langgraph.graph.message import AnyMessage, add_messages\n",
"from typing_extensions import TypedDict\n",
"\n",
"\n",
"class State(TypedDict):\n",
" messages: Annotated[list[AnyMessage], add_messages]\n",
"\n",
"\n",
"class Assistant:\n",
" def __init__(self, runnable: Runnable):\n",
" \"\"\"\n",
@@ -291,6 +294,7 @@
"source": [
"import uuid\n",
"\n",
"\n",
"def predict_react_agent_answer(example: dict):\n",
" \"\"\"Use this for answer evaluation\"\"\"\n",
"\n",