From fb5433bcca0335c8e3817f1f4c35e54362e0ea82 Mon Sep 17 00:00:00 2001 From: Harrison Chase Date: Wed, 16 Aug 2023 15:37:38 -0700 Subject: [PATCH] add web research notebook --- examples/web-research.ipynb | 465 ++++++++++++++++++++++++++++++++++++ 1 file changed, 465 insertions(+) create mode 100644 examples/web-research.ipynb diff --git a/examples/web-research.ipynb b/examples/web-research.ipynb new file mode 100644 index 000000000..8e35a17f0 --- /dev/null +++ b/examples/web-research.ipynb @@ -0,0 +1,465 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "cd80bc40-f10d-4ab3-826d-6cd0636d11e0", + "metadata": {}, + "outputs": [], + "source": [ + "from operator import itemgetter\n", + "\n", + "from langchain.chat_models import ChatOpenAI, ChatAnthropic\n", + "from langchain.prompts import SystemMessagePromptTemplate, ChatPromptTemplate\n", + "from langchain.schema.output_parser import StrOutputParser\n", + "from langchain.runnables.openai_functions import OpenAIFunctionsRouter\n", + "\n", + "from permchain.connection_inmemory import InMemoryPubSubConnection\n", + "from permchain.pubsub import PubSub\n", + "from permchain.topic import Topic" + ] + }, + { + "cell_type": "markdown", + "id": "54c553be-9ed1-452c-a4ab-828f34dbb3ce", + "metadata": {}, + "source": [ + "## Content Fetcher" + ] + }, + { + "cell_type": "markdown", + "id": "56a6788d-b67c-4331-af8d-7741a66f03af", + "metadata": {}, + "source": [ + "First, we are going to define our content fetcher. This is responsible for taking a search query an getting relevant web pages." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a9c32e92-6f19-4cf1-8b87-1b756de0e263", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.utilities import GoogleSearchAPIWrapper\n", + "from langchain.document_loaders import AsyncHtmlLoader\n", + "from langchain.document_transformers import Html2TextTransformer" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7db344e2-fac4-4047-a422-6b0df738e656", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# !pip install google-api-python-client\n", + "# !pip install html2text" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "25983034-a563-403e-b9d4-400a27e6ec29", + "metadata": {}, + "outputs": [], + "source": [ + "search_tool = GoogleSearchAPIWrapper()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "dcdc3812-0cdc-4677-bf9d-f9d7d5cac7e2", + "metadata": {}, + "outputs": [], + "source": [ + "def retrieve_documents(query):\n", + " query=query.strip().strip('\"')\n", + " urls_to_look = []\n", + " search_results = search_tool.results(query, 5)\n", + " for res in search_results:\n", + " if res.get(\"link\", None):\n", + " urls_to_look.append(res[\"link\"])\n", + " \n", + " # Relevant urls\n", + " # Load, split, and add new urls to vectorstore\n", + " if urls_to_look:\n", + " loader = AsyncHtmlLoader(urls_to_look)\n", + " html2text = Html2TextTransformer()\n", + " docs = loader.load()\n", + " docs = list(html2text.transform_documents(docs))\n", + " else:\n", + " docs = []\n", + " return docs" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d087d2b4-d1fa-4f63-81de-d3d4e1dc5b1b", + "metadata": {}, + "outputs": [], + "source": [ + "#docs = retrieve_documents(\"langchain\")" + ] + }, + { + "cell_type": "markdown", + "id": "d1c7d867-d9b3-4d05-826f-f9d6602c8be6", + "metadata": {}, + "source": [ + "## Querier\n", + "\n", + "We will now come up with an actor to generate a query to search for given a user request" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "f01c5fae-75d3-4107-81e6-9a254b0d50c5", + "metadata": {}, + "outputs": [], + "source": [ + "prompt = ChatPromptTemplate.from_template(\"Come up with a search query given the user question:\\n\\n{question}\")\n", + "query_chain = prompt | ChatOpenAI() | StrOutputParser()" + ] + }, + { + "cell_type": "markdown", + "id": "73e0b79f-f6bd-4f68-9433-645ee1eea81e", + "metadata": {}, + "source": [ + "## Summarizer\n", + "We will now come up with an actor to summarize the results given a query and some search results" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "901e1f8d-c973-4998-a731-5dab0c147b8c", + "metadata": {}, + "outputs": [], + "source": [ + "prompt = ChatPromptTemplate.from_template(\"Answer the user's question given the search results\\n\\n{question}{search_results}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "915bec33-d210-4471-b051-859ecba608be", + "metadata": {}, + "outputs": [], + "source": [ + "summarizer_chain = prompt | ChatOpenAI().with_fallbacks([ChatAnthropic(model=\"claude-2\")]) | StrOutputParser()" + ] + }, + { + "cell_type": "markdown", + "id": "6bf403ec-39a4-4061-9401-06850ffe3761", + "metadata": {}, + "source": [ + "## All together now!" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "edc0def4-d184-4438-9099-e6604ba9ff28", + "metadata": {}, + "outputs": [], + "source": [ + "query_inbox = Topic(\"query\")\n", + "summarizer_inbox = Topic(\"summarizer\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "33540bc4-bb75-4e5a-876f-2b7b842e5509", + "metadata": {}, + "outputs": [], + "source": [ + "query_actor = (\n", + " # Listed in inputs\n", + " Topic.IN.subscribe()\n", + " | query_chain\n", + " # The draft always goes to the editors inbox\n", + " | query_inbox.publish()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "23f89611-7b0f-4f01-b9a7-119124e3341d", + "metadata": {}, + "outputs": [], + "source": [ + "search_actor = (\n", + " query_inbox.subscribe()\n", + " | {\n", + " \"search_results\": retrieve_documents,\n", + " \"question\": lambda x: x,\n", + " }\n", + " | summarizer_inbox.publish()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "2d4a6b51-bd93-47c2-a301-9593a47df7d4", + "metadata": {}, + "outputs": [], + "source": [ + "summ_actor = (\n", + " summarizer_inbox.subscribe()\n", + " | summarizer_chain\n", + " | Topic.OUT.publish()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "11d3b066-7f95-4ad9-82d0-ba64bbf3e3fa", + "metadata": {}, + "outputs": [], + "source": [ + "web_researcher = PubSub(\n", + " processes=(query_actor, search_actor, summ_actor),\n", + " connection=InMemoryPubSubConnection(),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "bc022d51-69f0-4da9-8025-70afdc3cc6a8", + "metadata": {}, + "outputs": [], + "source": [ + "#web_researcher.invoke({\"question\": \"What is langsmith?\"})" + ] + }, + { + "cell_type": "markdown", + "id": "5952e9b8-2a56-4d2d-997a-dceb7652186f", + "metadata": {}, + "source": [ + "## Trying to use it as a sub component" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "43ca019d-a500-4c77-8e62-a46e54ffae7d", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.output_parsers.openai_functions import JsonKeyOutputFunctionsParser" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "858a82ae-a73f-4da2-9210-298af789ea30", + "metadata": {}, + "outputs": [], + "source": [ + "template = \"\"\"Write between 2 and 5 sub questions that serve as google search queries to search online that form an objective opinion from the following: {question}\"\"\"\n", + "functions = [\n", + " {\n", + " \"name\": \"sub_questions\",\n", + " \"description\": \"List of sub questions\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"questions\": {\n", + " \"type\": \"array\",\n", + " \"description\": \"List of sub questions to ask.\",\n", + " \"items\": {\n", + " \"type\": \"string\"\n", + " }\n", + " },\n", + " },\n", + " },\n", + " },\n", + "]\n", + "prompt = ChatPromptTemplate.from_template(template)\n", + "question_chain = prompt | ChatOpenAI(temperature=0).bind(functions=functions, function_call={\"name\":\"sub_questions\"}) | JsonKeyOutputFunctionsParser(key_name=\"questions\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "c0298fdc-0e7c-4e79-9cb7-cdd4d50fe88f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['What is the purpose of Langsmith?',\n", + " 'Who developed Langsmith?',\n", + " 'What are the key features of Langsmith?',\n", + " 'How does Langsmith work?',\n", + " 'Are there any alternatives to Langsmith?']" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "question_chain.invoke({\"question\": \"what is langsmith?\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "a0101bd0-cd95-4b13-ab26-d5d34d703bf9", + "metadata": {}, + "outputs": [], + "source": [ + "template = \"\"\"You are tasked with writing a research report to answer the following question:\n", + "\n", + "\n", + "{question}\n", + "\n", + "\n", + "In order to do that, you first came up with several sub questions and researched those. please find those below:\n", + "\n", + "\n", + "{research}\n", + "\n", + "\n", + "Now, write your final report answering the original question!\"\"\"\n", + "prompt = ChatPromptTemplate.from_template(template)\n", + "report_chain = prompt | ChatOpenAI() | StrOutputParser()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "d7715e0c-23c0-4985-97a7-bf5b151cf734", + "metadata": {}, + "outputs": [], + "source": [ + "research_inbox = Topic(\"research\")\n", + "writer_inbox = Topic(\"writer_inbox\")" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "f274027a-c9f0-4efa-b798-1921b9b376d9", + "metadata": {}, + "outputs": [], + "source": [ + "subquestion_actor = (\n", + " # Listed in inputs\n", + " Topic.IN.subscribe()\n", + " | question_chain\n", + " # The draft always goes to the editors inbox\n", + " | research_inbox.publish()\n", + ")\n", + "research_actor = (\n", + " research_inbox.subscribe()\n", + " | {\n", + " \"research\": lambda x: web_researcher.batch([{\"question\": i} for i in x]),\n", + " #\"research\": lambda x: [web_researcher.invoke({\"question\": i}) for i in x],\n", + " \"question\": Topic.IN.current() | itemgetter(\"question\"),\n", + " }\n", + " | writer_inbox.publish()\n", + ")\n", + "write_actor = (\n", + " writer_inbox.subscribe()\n", + " | report_chain\n", + " | Topic.OUT.publish()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "789b636d-23ce-44fb-a8e3-fdfbfabe77ff", + "metadata": {}, + "outputs": [], + "source": [ + "longer_researcher = PubSub(\n", + " processes=(subquestion_actor, research_actor, write_actor),\n", + " connection=InMemoryPubSubConnection(),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f713a31-5f60-4d90-9b95-3f42d8fbbddb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Fetching pages: 100%|#####################################################################################################################################################################################| 5/5 [00:01<00:00, 3.27it/s]\n", + "Fetching pages: 0%| | 0/5 [00:00