mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-09-12 04:37:51 +02:00
add web research notebook
This commit is contained in:
Harrison Chase
@@ -0,0 +1,465 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "cd80bc40-f10d-4ab3-826d-6cd0636d11e0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from operator import itemgetter\n",
|
||||
"\n",
|
||||
"from langchain.chat_models import ChatOpenAI, ChatAnthropic\n",
|
||||
"from langchain.prompts import SystemMessagePromptTemplate, ChatPromptTemplate\n",
|
||||
"from langchain.schema.output_parser import StrOutputParser\n",
|
||||
"from langchain.runnables.openai_functions import OpenAIFunctionsRouter\n",
|
||||
"\n",
|
||||
"from permchain.connection_inmemory import InMemoryPubSubConnection\n",
|
||||
"from permchain.pubsub import PubSub\n",
|
||||
"from permchain.topic import Topic"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "54c553be-9ed1-452c-a4ab-828f34dbb3ce",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Content Fetcher"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "56a6788d-b67c-4331-af8d-7741a66f03af",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"First, we are going to define our content fetcher. This is responsible for taking a search query an getting relevant web pages."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "a9c32e92-6f19-4cf1-8b87-1b756de0e263",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.utilities import GoogleSearchAPIWrapper\n",
|
||||
"from langchain.document_loaders import AsyncHtmlLoader\n",
|
||||
"from langchain.document_transformers import Html2TextTransformer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "7db344e2-fac4-4047-a422-6b0df738e656",
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# !pip install google-api-python-client\n",
|
||||
"# !pip install html2text"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"id": "25983034-a563-403e-b9d4-400a27e6ec29",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"search_tool = GoogleSearchAPIWrapper()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"id": "dcdc3812-0cdc-4677-bf9d-f9d7d5cac7e2",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def retrieve_documents(query):\n",
|
||||
" query=query.strip().strip('\"')\n",
|
||||
" urls_to_look = []\n",
|
||||
" search_results = search_tool.results(query, 5)\n",
|
||||
" for res in search_results:\n",
|
||||
" if res.get(\"link\", None):\n",
|
||||
" urls_to_look.append(res[\"link\"])\n",
|
||||
" \n",
|
||||
" # Relevant urls\n",
|
||||
" # Load, split, and add new urls to vectorstore\n",
|
||||
" if urls_to_look:\n",
|
||||
" loader = AsyncHtmlLoader(urls_to_look)\n",
|
||||
" html2text = Html2TextTransformer()\n",
|
||||
" docs = loader.load()\n",
|
||||
" docs = list(html2text.transform_documents(docs))\n",
|
||||
" else:\n",
|
||||
" docs = []\n",
|
||||
" return docs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "d087d2b4-d1fa-4f63-81de-d3d4e1dc5b1b",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#docs = retrieve_documents(\"langchain\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d1c7d867-d9b3-4d05-826f-f9d6602c8be6",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Querier\n",
|
||||
"\n",
|
||||
"We will now come up with an actor to generate a query to search for given a user request"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "f01c5fae-75d3-4107-81e6-9a254b0d50c5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prompt = ChatPromptTemplate.from_template(\"Come up with a search query given the user question:\\n\\n{question}\")\n",
|
||||
"query_chain = prompt | ChatOpenAI() | StrOutputParser()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "73e0b79f-f6bd-4f68-9433-645ee1eea81e",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Summarizer\n",
|
||||
"We will now come up with an actor to summarize the results given a query and some search results"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "901e1f8d-c973-4998-a731-5dab0c147b8c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prompt = ChatPromptTemplate.from_template(\"Answer the user's question given the search results\\n\\n<question>{question}</question><search_results>{search_results}</search_results>\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"id": "915bec33-d210-4471-b051-859ecba608be",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"summarizer_chain = prompt | ChatOpenAI().with_fallbacks([ChatAnthropic(model=\"claude-2\")]) | StrOutputParser()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6bf403ec-39a4-4061-9401-06850ffe3761",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## All together now!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"id": "edc0def4-d184-4438-9099-e6604ba9ff28",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query_inbox = Topic(\"query\")\n",
|
||||
"summarizer_inbox = Topic(\"summarizer\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"id": "33540bc4-bb75-4e5a-876f-2b7b842e5509",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"query_actor = (\n",
|
||||
" # Listed in inputs\n",
|
||||
" Topic.IN.subscribe()\n",
|
||||
" | query_chain\n",
|
||||
" # The draft always goes to the editors inbox\n",
|
||||
" | query_inbox.publish()\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "23f89611-7b0f-4f01-b9a7-119124e3341d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"search_actor = (\n",
|
||||
" query_inbox.subscribe()\n",
|
||||
" | {\n",
|
||||
" \"search_results\": retrieve_documents,\n",
|
||||
" \"question\": lambda x: x,\n",
|
||||
" }\n",
|
||||
" | summarizer_inbox.publish()\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"id": "2d4a6b51-bd93-47c2-a301-9593a47df7d4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"summ_actor = (\n",
|
||||
" summarizer_inbox.subscribe()\n",
|
||||
" | summarizer_chain\n",
|
||||
" | Topic.OUT.publish()\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"id": "11d3b066-7f95-4ad9-82d0-ba64bbf3e3fa",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"web_researcher = PubSub(\n",
|
||||
" processes=(query_actor, search_actor, summ_actor),\n",
|
||||
" connection=InMemoryPubSubConnection(),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"id": "bc022d51-69f0-4da9-8025-70afdc3cc6a8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#web_researcher.invoke({\"question\": \"What is langsmith?\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5952e9b8-2a56-4d2d-997a-dceb7652186f",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Trying to use it as a sub component"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"id": "43ca019d-a500-4c77-8e62-a46e54ffae7d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from langchain.output_parsers.openai_functions import JsonKeyOutputFunctionsParser"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"id": "858a82ae-a73f-4da2-9210-298af789ea30",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"template = \"\"\"Write between 2 and 5 sub questions that serve as google search queries to search online that form an objective opinion from the following: {question}\"\"\"\n",
|
||||
"functions = [\n",
|
||||
" {\n",
|
||||
" \"name\": \"sub_questions\",\n",
|
||||
" \"description\": \"List of sub questions\",\n",
|
||||
" \"parameters\": {\n",
|
||||
" \"type\": \"object\",\n",
|
||||
" \"properties\": {\n",
|
||||
" \"questions\": {\n",
|
||||
" \"type\": \"array\",\n",
|
||||
" \"description\": \"List of sub questions to ask.\",\n",
|
||||
" \"items\": {\n",
|
||||
" \"type\": \"string\"\n",
|
||||
" }\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
"]\n",
|
||||
"prompt = ChatPromptTemplate.from_template(template)\n",
|
||||
"question_chain = prompt | ChatOpenAI(temperature=0).bind(functions=functions, function_call={\"name\":\"sub_questions\"}) | JsonKeyOutputFunctionsParser(key_name=\"questions\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"id": "c0298fdc-0e7c-4e79-9cb7-cdd4d50fe88f",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"['What is the purpose of Langsmith?',\n",
|
||||
" 'Who developed Langsmith?',\n",
|
||||
" 'What are the key features of Langsmith?',\n",
|
||||
" 'How does Langsmith work?',\n",
|
||||
" 'Are there any alternatives to Langsmith?']"
|
||||
]
|
||||
},
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"question_chain.invoke({\"question\": \"what is langsmith?\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"id": "a0101bd0-cd95-4b13-ab26-d5d34d703bf9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"template = \"\"\"You are tasked with writing a research report to answer the following question:\n",
|
||||
"\n",
|
||||
"<question>\n",
|
||||
"{question}\n",
|
||||
"</question>\n",
|
||||
"\n",
|
||||
"In order to do that, you first came up with several sub questions and researched those. please find those below:\n",
|
||||
"\n",
|
||||
"<research>\n",
|
||||
"{research}\n",
|
||||
"</research>\n",
|
||||
"\n",
|
||||
"Now, write your final report answering the original question!\"\"\"\n",
|
||||
"prompt = ChatPromptTemplate.from_template(template)\n",
|
||||
"report_chain = prompt | ChatOpenAI() | StrOutputParser()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"id": "d7715e0c-23c0-4985-97a7-bf5b151cf734",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"research_inbox = Topic(\"research\")\n",
|
||||
"writer_inbox = Topic(\"writer_inbox\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 34,
|
||||
"id": "f274027a-c9f0-4efa-b798-1921b9b376d9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"subquestion_actor = (\n",
|
||||
" # Listed in inputs\n",
|
||||
" Topic.IN.subscribe()\n",
|
||||
" | question_chain\n",
|
||||
" # The draft always goes to the editors inbox\n",
|
||||
" | research_inbox.publish()\n",
|
||||
")\n",
|
||||
"research_actor = (\n",
|
||||
" research_inbox.subscribe()\n",
|
||||
" | {\n",
|
||||
" \"research\": lambda x: web_researcher.batch([{\"question\": i} for i in x]),\n",
|
||||
" #\"research\": lambda x: [web_researcher.invoke({\"question\": i}) for i in x],\n",
|
||||
" \"question\": Topic.IN.current() | itemgetter(\"question\"),\n",
|
||||
" }\n",
|
||||
" | writer_inbox.publish()\n",
|
||||
")\n",
|
||||
"write_actor = (\n",
|
||||
" writer_inbox.subscribe()\n",
|
||||
" | report_chain\n",
|
||||
" | Topic.OUT.publish()\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 35,
|
||||
"id": "789b636d-23ce-44fb-a8e3-fdfbfabe77ff",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"longer_researcher = PubSub(\n",
|
||||
" processes=(subquestion_actor, research_actor, write_actor),\n",
|
||||
" connection=InMemoryPubSubConnection(),\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8f713a31-5f60-4d90-9b95-3f42d8fbbddb",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Fetching pages: 100%|#####################################################################################################################################################################################| 5/5 [00:01<00:00, 3.27it/s]\n",
|
||||
"Fetching pages: 0%| | 0/5 [00:00<?, ?it/s]\n",
|
||||
"Fetching pages: 0%| | 0/5 [00:00<?, ?it/s]\u001b[A\n",
|
||||
"\n",
|
||||
"Fetching pages: 0%| | 0/5 [00:00<?, ?it/s]\u001b[A\u001b[A\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Fetching pages: 0%| | 0/5 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Fetching pages: 100%|#####################################################################################################################################################################################| 5/5 [00:01<00:00, 3.88it/s]\u001b[A\u001b[A\u001b[A\u001b[A\n",
|
||||
"Fetching pages: 100%|#####################################################################################################################################################################################| 5/5 [00:00<00:00, 9.07it/s]\n",
|
||||
"Fetching pages: 100%|#####################################################################################################################################################################################| 5/5 [00:00<00:00, 6.00it/s]\n",
|
||||
"\n",
|
||||
"Fetching pages: 100%|#####################################################################################################################################################################################| 5/5 [00:01<00:00, 4.79it/s]\u001b[A\n",
|
||||
"\n",
|
||||
"Fetching pages: 100%|#####################################################################################################################################################################################| 5/5 [00:00<00:00, 8.80it/s]\u001b[A\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"longer_researcher.invoke({\"question\": \"what is langsmith?\"})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fde8b227-64ec-4e96-b337-fc888e7ad787",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.1"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Reference in New Issue
Block a user