From b721aad3e48850d7fb7bb30ea77178798f773ef7 Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:06:58 -0800 Subject: [PATCH] Update notebook --- examples/storm/storm.ipynb | 741 ++++++++++++++++++++++++++++--------- 1 file changed, 572 insertions(+), 169 deletions(-) diff --git a/examples/storm/storm.ipynb b/examples/storm/storm.ipynb index c5b294ac8..b4191e6bd 100644 --- a/examples/storm/storm.ipynb +++ b/examples/storm/storm.ipynb @@ -38,6 +38,18 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, + "outputs": [], + "source": [ + "from langchain_openai import ChatOpenAI\n", + "\n", + "fast_llm = ChatOpenAI(model=\"gpt-3.5-turbo\")\n", + "long_context_llm = ChatOpenAI(model=\"gpt-4-turbo-preview\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, "outputs": [ { "name": "stderr", @@ -49,7 +61,6 @@ } ], "source": [ - "from langchain_openai import ChatOpenAI\n", "from langchain_core.pydantic_v1 import BaseModel, Field\n", "from typing import List, Optional\n", "from langchain_core.prompts import ChatPromptTemplate\n", @@ -86,7 +97,7 @@ " def as_str(self) -> str:\n", " subsections = \"\\n\\n\".join(\n", " f\"### {subsection.subsection_title}\\n\\n{subsection.description}\"\n", - " for subsection in self.subsections or []\n", + " for subsection in self.subsections or []\n", " )\n", " return f\"## {self.section_title}\\n\\n{self.description}\\n\\n{subsections}\".strip()\n", "\n", @@ -94,7 +105,8 @@ "class Outline(BaseModel):\n", " page_title: str = Field(..., title=\"Title of the Wikipedia page\")\n", " sections: List[Section] = Field(\n", - " default_factory=list, title=\"Titles and descriptions for each section of the Wikipedia page.\"\n", + " default_factory=list,\n", + " title=\"Titles and descriptions for each section of the Wikipedia page.\",\n", " )\n", "\n", " @property\n", @@ -103,14 +115,14 @@ " return f\"# {self.page_title}\\n\\n{sections}\".strip()\n", "\n", "\n", - "generate_outline_direct = direct_gen_outline_prompt | ChatOpenAI(\n", - " model=\"gpt-3.5-turbo\"\n", - ").with_structured_output(Outline)" + "generate_outline_direct = direct_gen_outline_prompt | fast_llm.with_structured_output(\n", + " Outline\n", + ")" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -121,19 +133,23 @@ "\n", "## Introduction\n", "\n", - "Overview of million-plus token context window language models and RAG (Retrieval-Augmented Generation).\n", + "Overview of million-plus token context window language models and the RAG (Retrieval-Augmented Generation) architecture.\n", "\n", - "## Million-Plus Token Context Window Language Models\n", + "## Benefits of Million-Plus Token Context Window Language Models\n", "\n", - "Explanation of million-plus token context window language models, including architecture, training data, benefits, and challenges.\n", + "Discuss the advantages of using million-plus token context window language models in natural language processing tasks.\n", "\n", - "## RAG (Retrieval-Augmented Generation)\n", + "## Challenges of Million-Plus Token Context Window Language Models\n", "\n", - "Explanation of RAG, its components, and how it integrates with million-plus token context window language models.\n", + "Explore the limitations and obstacles associated with million-plus token context window language models.\n", "\n", - "## Impact on RAG\n", + "## Integration of Million-Plus Token Models with RAG\n", "\n", - "Discuss the implications of using million-plus token context window language models with RAG, including improvements in performance, challenges, and future research directions.\n" + "Examine how million-plus token context window language models can be integrated with the RAG architecture for improved performance.\n", + "\n", + "## Applications of RAG with Million-Plus Token Models\n", + "\n", + "Highlight the potential applications and use cases of combining RAG with million-plus token context window language models.\n" ] } ], @@ -158,7 +174,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -171,30 +187,30 @@ "\"\"\"\n", ")\n", "\n", + "\n", "class RelatedSubjects(BaseModel):\n", " topics: List[str] = Field(\n", " description=\"Comprehensive list of related subjects as background research.\",\n", " )\n", "\n", "\n", - "expand_chain = (\n", - " gen_related_topics_prompt \n", - " | ChatOpenAI(model=\"gpt-3.5-turbo\").with_structured_output(RelatedSubjects)\n", - ")\n" + "expand_chain = gen_related_topics_prompt | fast_llm.with_structured_output(\n", + " RelatedSubjects\n", + ")" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "RelatedSubjects(topics=['Impact of million-plus token context window language models', 'RAG'])" + "RelatedSubjects(topics=['million-plus token context window language models', 'Retriever-Reader-Generator (RAG) model', 'Impact of RAG on language understanding'])" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -206,7 +222,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -257,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -281,7 +297,9 @@ "@as_runnable\n", "async def survey_subjects(topic: str):\n", " related_subjects = await expand_chain.ainvoke({\"topic\": topic})\n", - " retrieved_docs = await wikipedia_retriever.abatch(related_subjects.topics, return_exceptions=True)\n", + " retrieved_docs = await wikipedia_retriever.abatch(\n", + " related_subjects.topics, return_exceptions=True\n", + " )\n", " all_docs = []\n", " for docs in retrieved_docs:\n", " if isinstance(docs, BaseException):\n", @@ -293,44 +311,52 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/wfh/code/lc/community/langgraph-engineer/.venv/lib/python3.11/site-packages/wikipedia/wikipedia.py:389: GuessedAtParserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n", + "\n", + "The code that caused this warning is on line 389 of the file /Users/wfh/code/lc/community/langgraph-engineer/.venv/lib/python3.11/site-packages/wikipedia/wikipedia.py. To get rid of this warning, pass the additional argument 'features=\"lxml\"' to the BeautifulSoup constructor.\n", + "\n", + " lis = BeautifulSoup(html).find_all('li')\n" + ] + } + ], "source": [ "perspectives = await survey_subjects.ainvoke(example_topic)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'editors': [{'affiliation': 'Language model research institute',\n", - " 'name': 'Alice',\n", - " 'role': 'Language model researcher',\n", - " 'description': 'Alice is a language model researcher focusing on the impact of million-plus token context window language models on RAG. She is interested in analyzing the advancements in language models and their implications on the RAG (Retrieve, Analyze, Generate) framework.'},\n", - " {'affiliation': 'Vector database company',\n", - " 'name': 'Bob',\n", - " 'role': 'Vector database expert',\n", - " 'description': 'Bob is a vector database expert who will provide insights on how vector databases can support the storage and retrieval of large language models like the million-plus token context window language models used in RAG systems.'},\n", - " {'affiliation': 'Natural language processing organization',\n", - " 'name': 'Charlie',\n", - " 'role': 'NLP specialist',\n", - " 'description': 'Charlie is an NLP specialist who will focus on the application of million-plus token context window language models in natural language processing tasks, particularly in the context of the RAG framework.'},\n", - " {'affiliation': 'Machine learning consultancy',\n", - " 'name': 'David',\n", - " 'role': 'Machine learning consultant',\n", - " 'description': 'David is a machine learning consultant with expertise in training and optimizing large language models. He will discuss the machine learning techniques involved in developing million-plus token context window models for RAG.'},\n", - " {'affiliation': 'Artificial intelligence company',\n", - " 'name': 'Eve',\n", - " 'role': 'AI expert',\n", - " 'description': 'Eve is an AI expert who will provide insights on the broader implications of million-plus token context window language models in artificial intelligence applications, including their impact on the RAG framework.'}]}" + "{'editors': [{'affiliation': 'Research Institution',\n", + " 'name': 'Dr. Researcher',\n", + " 'role': 'Researcher',\n", + " 'description': 'Dr. Researcher will focus on analyzing the impact of million-plus token context window language models on the RAG (Retrieval-Augmented Generation) framework, specifically looking at the efficiency, effectiveness, and potential challenges that arise from integrating such large language models into the RAG framework.'},\n", + " {'affiliation': 'Tech Company',\n", + " 'name': 'AI Engineer',\n", + " 'role': 'AI Engineer',\n", + " 'description': 'AI Engineer will provide insights into the technical aspects of implementing million-plus token context window language models within the RAG framework. They will focus on the practical challenges, optimizations, and enhancements needed to leverage these models effectively in the RAG framework.'},\n", + " {'affiliation': 'Academic Institution',\n", + " 'name': 'Prof. Linguist',\n", + " 'role': 'Linguist',\n", + " 'description': 'Prof. Linguist will examine the linguistic implications of using million-plus token context window language models in the RAG framework. They will explore how such large models affect language generation, coherence, and understanding within the RAG context.'},\n", + " {'affiliation': 'Industry',\n", + " 'name': 'Content Creator',\n", + " 'role': 'Content Creator',\n", + " 'description': 'Content Creator will provide a creative perspective on the impact of million-plus token context window language models on the RAG framework. They will focus on storytelling, narrative quality, and the potential for generating engaging content using these large language models.'}]}" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -350,7 +376,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -390,7 +416,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -440,27 +466,25 @@ " gn_chain = (\n", " RunnableLambda(swap_roles).bind(name=editor.name)\n", " | gen_qn_prompt.partial(persona=editor.persona)\n", - " | ChatOpenAI(model=\"gpt-3.5-turbo\")\n", + " | fast_llm\n", " | RunnableLambda(tag_with_name).bind(name=editor.name)\n", " )\n", " result = await gn_chain.ainvoke(state)\n", - " return {\n", - " \"messages\": [result]\n", - " }" + " return {\"messages\": [result]}" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "\"Yes, that's correct. I am focusing on the impact of million-plus token context window language models on the RAG framework. These language models have significantly expanded the scope of information that can be considered when retrieving, analyzing, and generating content. Is there a specific aspect of this topic that you would like to explore further?\"" + "\"Yes, that's correct. I am researching the impact of million-plus token context window language models on the RAG (Retrieval-Augmented Generation) framework. I am particularly interested in understanding how these large language models affect the efficiency, effectiveness, and any potential challenges that may arise when integrating them into the RAG framework. Is there any specific aspect of this topic that you would like to know more about?\"" ] }, - "execution_count": 11, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -481,7 +505,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -507,31 +531,32 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['Impact of million-plus token context window language models on the RAG framework',\n", - " 'Benefits of using million-plus token context window language models in the RAG framework',\n", - " 'Challenges of integrating million-plus token context window language models with the RAG framework',\n", - " 'Comparison of different million-plus token context window language models in the RAG framework']" + "['Impact of million-plus token context window language models on RAG framework efficiency',\n", + " 'Effectiveness of large language models in RAG framework',\n", + " 'Challenges of integrating million-plus token context window language models into RAG framework']" ] }, - "execution_count": 13, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "queries = await gen_queries_chain.ainvoke({\"messages\": [HumanMessage(content=question[\"messages\"][0].content)]})\n", - "queries['parsed'].queries" + "queries = await gen_queries_chain.ainvoke(\n", + " {\"messages\": [HumanMessage(content=question[\"messages\"][0].content)]}\n", + ")\n", + "queries[\"parsed\"].queries" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -545,19 +570,28 @@ "\n", " @property\n", " def as_str(self) -> str:\n", - " return f\"{self.answer}\\n\\nCitations:\\n\\n\" + \"\\n\".join(f\"[{i+1}]: {url}\" for i, url in enumerate(self.cited_urls))\n", + " return f\"{self.answer}\\n\\nCitations:\\n\\n\" + \"\\n\".join(\n", + " f\"[{i+1}]: {url}\" for i, url in enumerate(self.cited_urls)\n", + " )\n", + "\n", "\n", "gen_answer_prompt = ChatPromptTemplate.from_messages(\n", - " [(\"system\", \"\"\"You are an expert who can use information effectively. You are chatting with a Wikipedia writer who wants\\\n", + " [\n", + " (\n", + " \"system\",\n", + " \"\"\"You are an expert who can use information effectively. You are chatting with a Wikipedia writer who wants\\\n", " to write a Wikipedia page on the topic you know. You have gathered the related information and will now use the information to form a response.\n", "\n", "Make your response as informative as possible and make sure every sentence is supported by the gathered information.\n", - "Each response must be backed up by a citation from a reliable source, formatted as a footnote, reproducing the URLS after your response.\"\"\"),\n", - " MessagesPlaceholder(variable_name=\"messages\", optional=True),\n", + "Each response must be backed up by a citation from a reliable source, formatted as a footnote, reproducing the URLS after your response.\"\"\",\n", + " ),\n", + " MessagesPlaceholder(variable_name=\"messages\", optional=True),\n", " ]\n", ")\n", "\n", - "gen_answer_chain = gen_answer_prompt | ChatOpenAI(model=\"gpt-3.5-turbo\").with_structured_output(AnswerWithCitations, include_raw=True)" + "gen_answer_chain = gen_answer_prompt | fast_llm.with_structured_output(\n", + " AnswerWithCitations, include_raw=True\n", + ")" ] }, { @@ -572,26 +606,27 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ "from langchain_community.utilities.duckduckgo_search import DuckDuckGoSearchAPIWrapper\n", + "\n", "search_engine = DuckDuckGoSearchAPIWrapper()\n", "from langchain_core.tools import tool\n", + "\n", + "\n", "# TODO: remove when i get my api limit bumped\n", "@tool\n", "async def search_engine(query: str):\n", " \"\"\"Search engine to the internet.\"\"\"\n", - " results = DuckDuckGoSearchAPIWrapper()._ddgs_text(\"beijing olympics\")\n", - " return [\n", - " {\"content\": r[\"body\"],\n", - " \"url\": r[\"href\"]} for r in results]\n" + " results = DuckDuckGoSearchAPIWrapper()._ddgs_text(query)\n", + " return [{\"content\": r[\"body\"], \"url\": r[\"href\"]} for r in results]" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -610,18 +645,21 @@ "):\n", " swapped_state = swap_roles(state, name) # Convert all other AI messages\n", " queries = await gen_queries_chain.ainvoke(swapped_state)\n", - " query_results = await search_engine.abatch(queries[\"parsed\"].queries, config, return_exceptions=True)\n", - " successful_results = [res for res in query_results if not isinstance(res, Exception)]\n", - " all_query_results = {res[\"url\"]: res[\"content\"] for results in successful_results for res in results}\n", + " query_results = await search_engine.abatch(\n", + " queries[\"parsed\"].queries, config, return_exceptions=True\n", + " )\n", + " successful_results = [\n", + " res for res in query_results if not isinstance(res, Exception)\n", + " ]\n", + " all_query_results = {\n", + " res[\"url\"]: res[\"content\"] for results in successful_results for res in results\n", + " }\n", " # We could be more precise about handling max token length if we wanted to here\n", " dumped = json.dumps(all_query_results)[:max_str_len]\n", " ai_message: AIMessage = queries[\"raw\"]\n", " tool_call = queries[\"raw\"].additional_kwargs[\"tool_calls\"][0]\n", " tool_id = tool_call[\"id\"]\n", - " tool_message = ToolMessage(\n", - " tool_call_id=tool_id,\n", - " content=dumped\n", - " )\n", + " tool_message = ToolMessage(tool_call_id=tool_id, content=dumped)\n", " swapped_state[\"messages\"].extend([ai_message, tool_message])\n", " # Only update the shared state with the final answer to avoid\n", " # polluting the dialogue history with intermediate messages\n", @@ -630,24 +668,21 @@ " # Save the retrieved information to a the shared state for future reference\n", " cited_references = {k: v for k, v in all_query_results.items() if k in cited_urls}\n", " formatted_message = AIMessage(name=name, content=generated[\"parsed\"].as_str)\n", - " return {\n", - " \"messages\": [formatted_message],\n", - " \"references\": cited_references\n", - " }" + " return {\"messages\": [formatted_message], \"references\": cited_references}" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'Million-plus token context window language models have revolutionized the RAG (Retrieval-Augmented Generation) framework by significantly expanding the scope of information considered during content retrieval, analysis, and generation. These advanced language models, with their extensive context windows, allow for more nuanced understanding of text and context, leading to more accurate and contextually relevant content generation within the RAG framework.\\n\\nCitations:\\n\\n[1]: https://www.teamusa.com/olympic-games-beijing-2022\\n[2]: https://apnews.com/article/beijing-olympics-nhl-d32ef9ddd57b6be68f3ae5b55b47c3c4\\n[3]: https://www.britannica.com/topic/2008-Beijing-Olympic-Games-1702245\\n[4]: https://olympics.com/ioc/news/final-report-highlights-legacy-of-olympic-winter-games-beijing-2022'" + "'Large language models with million-plus token context windows, such as Gemini 1.5, have generated discussions in the AI community regarding their impact on the Retrieval-Augmented Generation (RAG) framework. These models are believed to potentially have a negative effect on RAG^(1). The RAG framework typically involves components like Milvus as the vector database, LangChain as the orchestrator, and large language models like GTE-Large for text generation^(2). While large context windows are desirable in language models, the high fine-tuning costs, scarcity of long texts, and challenges like catastrophic values introduced by new token positions limit the current extended context windows to around 128k tokens^(3). Recent advancements, like LongRoPE, have extended the context window of pre-trained large language models significantly to 2048k tokens^(3). The integration of retrieval mechanisms with long context language models, such as GPT-3.5-Turbo-16k and Llama2-7B-chat-4k, has been explored to evaluate the impact of retrieval on model performance in the RAG framework^(4). Challenges in integrating large language models into RAG include inaccuracies, cost-efficiency, and the need for optimization through vector databases^(5). RAG combines large language models with retrieval modules to ground text generation in knowledge, addressing challenges like inaccuracies and cost-efficiency^(6). The RAG framework involves indexing external data sources, converting them into vector embeddings, and retrieval processes to enhance the capabilities of large language models like GPT-3 or BERT^(7). While RAG offers flexibility and scalability in integrating with existing AI systems, challenges persist in ensuring the accuracy of external data sources^(8). Large language models like GPT-4 have demonstrated impressive text generation abilities, but challenges remain in retaining factual knowledge^(9).\\n\\nCitations:\\n\\n[1]: https://medium.com/enterprise-rag/why-gemini-1-5-and-other-large-context-models-are-bullish-for-rag-ce3218930bb4\\n[2]: https://zilliz.com/blog/building-rag-without-openai-mixtral-milvus-octoai\\n[3]: https://arxiv.org/abs/2402.13753\\n[4]: https://blog.llamaindex.ai/nvidia-research-rag-with-long-context-llms-7d94d40090c4\\n[5]: https://www.infoworld.com/article/3712227/what-is-rag-more-accurate-and-reliable-llms.html\\n[6]: https://medium.com/@juanc.olamendy/rag-best-practices-enhancing-large-language-models-with-retrieval-augmented-generation-6961c8b834ff\\n[7]: https://www.deepset.ai/blog/generative-llm-evaluation-rag\\n[8]: https://ai88.substack.com/p/rag-vs-context-window-in-gpt4-accuracy-cost\\n[9]: https://wetheitguys.medium.com/understanding-retrieval-augmented-generation-rag-with-large-language-models-llms-b77c76b9f9d8\\n[10]: https://medium.com/@bijit211987/optimizing-rag-for-llms-apps-53f6056d8118'" ] }, - "execution_count": 17, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -656,12 +691,12 @@ "example_answer = await gen_answer(\n", " {\"messages\": [HumanMessage(content=question[\"messages\"][0].content)]}\n", ")\n", - "example_answer['messages'][-1].content" + "example_answer[\"messages\"][-1].content" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -680,6 +715,7 @@ " return END\n", " return \"ask_question\"\n", "\n", + "\n", "builder = StateGraph(InterviewState)\n", "\n", "builder.add_node(\"ask_question\", generate_question)\n", @@ -693,7 +729,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -701,9 +737,23 @@ "output_type": "stream", "text": [ "ask_question\n", - "-- [AIMessage(content=\"Yes, that's correct. I am interested in understanding how million-plus token context window language models are influencing the RAG (Retrieve, Analyze, Generate) framework. Do you have insights on how these advanced language models are changing the way information is retrieved, a\n", + "-- [AIMessage(content=\"Yes, that's correct. I am focusing on analyzing the impact of million-plus token context window language models on the RAG (Retrieval-Augmented Generation) framework. This involves looking at how these large language models affect the efficiency, effectiveness, and potential chal\n", "answer_question\n", - "-- [AIMessage(content='Million-plus token context window language models have significantly impacted the RAG (Retrieve, Analyze, Generate) framework in natural language processing tasks. These advanced language models, such as GPT-3 with 175 billion parameters, have revolutionized the way information i\n" + "-- [AIMessage(content='The impact of million-plus token context window language models on the RAG (Retrieval-Augmented Generation) framework has been a topic of discussion in the AI community. For example, the introduction of Gemini 1.5, with a 1 million token context window, has raised concerns about \n", + "ask_question\n", + "-- [AIMessage(content='Thank you for providing such detailed information and relevant citations. Could you elaborate on the specific challenges researchers have encountered when integrating million-plus token context window language models into the RAG framework, and how they have attempted to address \n", + "answer_question\n", + "-- [AIMessage(content=\"Integrating million-plus token context window language models into the RAG (Retrieval-Augmented Generation) framework poses challenges such as high fine-tuning costs, scarcity of long texts, and the introduction of catastrophic values by new token positions, limiting the extended\n", + "ask_question\n", + "-- [AIMessage(content='Thank you for sharing this insightful information about the challenges and advancements in integrating million-plus token context window language models into the RAG framework. Can you provide more details on how RAG leverages external knowledge sources through retrieval to enhan\n", + "answer_question\n", + "-- [AIMessage(content='Retrieval-Augmented Generation (RAG) is a technique that enhances the accuracy and reliability of generative AI models, such as Large Language Models (LLMs), by incorporating facts fetched from external sources. RAG leverages external knowledge sources through retrieval to addres\n", + "ask_question\n", + "-- [AIMessage(content='Thank you for providing a concise summary of how Retrieval-Augmented Generation (RAG) leverages external knowledge sources through retrieval to enhance the accuracy and credibility of Large Language Models (LLMs). This information will be valuable for my research and editing of t\n", + "answer_question\n", + "-- [AIMessage(content='RAG leverages external knowledge sources through retrieval to address challenges like hallucination, outdated knowledge, and non-transparent reasoning processes. By integrating information from external databases, RAG improves the credibility and accuracy of LLMs, making them mor\n", + "__end__\n", + "-- [AIMessage(content='So you said you were writing an article on Impact of million-plus token context window language models on RAG?', name='Subject Matter Expert'), AIMessage(content=\"Yes, that's correct. I am focusing on analyzing the impact of million-plus token context window language models on th\n" ] } ], @@ -712,19 +762,24 @@ "\n", "initial_state = {\n", " \"editor\": perspectives.editors[0],\n", - " \"messages\": [AIMessage(content=f\"So you said you were writing an article on {example_topic}?\", name=\"Subject Matter Expert\")]\n", + " \"messages\": [\n", + " AIMessage(\n", + " content=f\"So you said you were writing an article on {example_topic}?\",\n", + " name=\"Subject Matter Expert\",\n", + " )\n", + " ],\n", "}\n", "async for step in interview_graph.astream(initial_state):\n", " name = next(iter(step))\n", " print(name)\n", - " print(\"-- \", str(step[name]['messages'])[:300])\n", + " print(\"-- \", str(step[name][\"messages\"])[:300])\n", " if END in step:\n", " final_step = step" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -742,7 +797,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -755,17 +810,22 @@ "Topic you are writing about: {topic}\\\n", "Old outline: {old_outline}\"\"\",\n", " ),\n", - " (\"user\", \"Refine the outline based on your conversations with subject-matter experts:\\n\\nConversations:\\n\\n{conversations}\\n\\nWrite the refined Wikipedia outline:\"),\n", + " (\n", + " \"user\",\n", + " \"Refine the outline based on your conversations with subject-matter experts:\\n\\nConversations:\\n\\n{conversations}\\n\\nWrite the refined Wikipedia outline:\",\n", + " ),\n", " ]\n", ")\n", "\n", "# Using turbo preview since the context can get quite long\n", - "refine_outline_chain = refine_outline_prompt | ChatOpenAI(model=\"gpt-4-turbo-preview\").with_structured_output(Outline)\n" + "refine_outline_chain = refine_outline_prompt | long_context_llm.with_structured_output(\n", + " Outline\n", + ")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -782,9 +842,65 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 38, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Impact of Million-Plus Token Context Window Language Models on RAG\n", + "\n", + "## Introduction\n", + "\n", + "An overview of the development and significance of million-plus token context window language models and the concept of Retrieval-Augmented Generation (RAG).\n", + "\n", + "## The Evolution of Large Context Windows in Language Models\n", + "\n", + "A historical perspective on the growth of context window sizes in language models, including key milestones such as Gemini 1.5, Mixtral, GPT-3.5-Turbo-16k, Llama2-7B-chat-4k, and LongRoPE.\n", + "\n", + "### Key Milestones\n", + "\n", + "Discussion of significant advancements and models that have shaped the current landscape of large context window language models.\n", + "\n", + "### Challenges Overcome\n", + "\n", + "Examination of the technical and theoretical hurdles encountered in expanding the context window sizes of language models.\n", + "\n", + "## Integration Challenges with RAG\n", + "\n", + "Detailed analysis of the specific challenges faced when integrating million-plus token context window language models into the RAG framework, such as high fine-tuning costs, scarcity of long texts, and catastrophic values introduced by new token positions.\n", + "\n", + "### Addressing High Fine-Tuning Costs\n", + "\n", + "Exploration of strategies and technological advancements aimed at reducing the high fine-tuning costs associated with large context windows.\n", + "\n", + "### Overcoming Scarcity of Long Texts\n", + "\n", + "Discussion on methods to mitigate the scarcity of long texts suitable for training and leveraging million-plus token models.\n", + "\n", + "### Mitigating Catastrophic Values\n", + "\n", + "Analysis of approaches to limit the impact of catastrophic values introduced by new token positions in extended context windows.\n", + "\n", + "## Enhancing RAG with External Knowledge Sources\n", + "\n", + "Insight into how RAG leverages external knowledge sources through retrieval to address challenges like hallucination, outdated knowledge, and non-transparent reasoning processes, thereby enhancing the accuracy and credibility of LLMs.\n", + "\n", + "### Improving Credibility and Accuracy\n", + "\n", + "Discussion on the importance of integrating external data sources to enhance the credibility and accuracy of responses generated by LLMs.\n", + "\n", + "### Addressing Hallucination and Outdated Knowledge\n", + "\n", + "Analysis of how RAG's retrieval mechanism helps in minimizing issues of hallucination and outdated knowledge by providing up-to-date and relevant information.\n", + "\n", + "## Applications and Future Directions\n", + "\n", + "Exploration of potential applications for RAG integrated with million-plus token context window language models and speculation on future developments in this area.\n" + ] + } + ], "source": [ "print(refined_outline.as_str)" ] @@ -800,17 +916,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "class SubSection(BaseModel):\n", " subsection_title: str = Field(..., title=\"Title of the subsection\")\n", - " content: str = Field(..., title=\"Full content of the subsection. Include [#] citations to the cited sources where relevant.\")\n", + " content: str = Field(\n", + " ...,\n", + " title=\"Full content of the subsection. Include [#] citations to the cited sources where relevant.\",\n", + " )\n", "\n", " @property\n", " def as_str(self) -> str:\n", - " \n", " return f\"### {self.subsection_title}\\n\\n{self.content}\".strip()\n", "\n", "\n", @@ -826,16 +944,18 @@ " @property\n", " def as_str(self) -> str:\n", " subsections = \"\\n\\n\".join(\n", - " subsection.as_str\n", - " for subsection in self.subsections or []\n", + " subsection.as_str for subsection in self.subsections or []\n", " )\n", " citations = \"\\n\".join([f\" [{i}] {cit}\" for i, cit in enumerate(self.citations)])\n", - " return f\"## {self.section_title}\\n\\n{self.content}\\n\\n{subsections}\".strip() + f\"\\n\\n{citations}\".strip()" + " return (\n", + " f\"## {self.section_title}\\n\\n{self.content}\\n\\n{subsections}\".strip()\n", + " + f\"\\n\\n{citations}\".strip()\n", + " )" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -845,7 +965,10 @@ "from langchain_openai import OpenAIEmbeddings\n", "\n", "embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\")\n", - "reference_docs = [Document(page_content=v, metadata={\"source\": k}) for k, v in final_state[\"references\"].items()]\n", + "reference_docs = [\n", + " Document(page_content=v, metadata={\"source\": k})\n", + " for k, v in final_state[\"references\"].items()\n", + "]\n", "# This really doesn't need to be a vectorstore.\n", "# could just be a numpy matrix\n", "vectorstore = SKLearnVectorStore.from_documents(\n", @@ -857,54 +980,141 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(page_content='Large Language Models (LLMs) have achieved remarkable success across various tasks. However, they often grapple with a limited context window size due to the high costs of fine-tuning, scarcity of lengthy texts, and the introduction of catastrophic values by new token positions. To address this issue, in a new paper LongRoPE: Extending LLM Context Window', metadata={'id': '20dbbce3-ae12-4a05-94e9-df4a97676098', 'source': 'https://syncedreview.com/2024/02/25/microsofts-longrope-breaks-the-limit-of-context-window-of-llms-extents-it-to-2-million-tokens/'}),\n", + " Document(page_content='Large context window is a desirable feature in large language models (LLMs). However, due to high fine-tuning costs, scarcity of long texts, and catastrophic values introduced by new token positions, current extended context windows are limited to around 128k tokens. This paper introduces LongRoPE that, for the first time, extends the context window of pre-trained LLMs to an impressive 2048k ...', metadata={'id': 'f7881b24-697b-46fa-a31c-8e476ae40f3f', 'source': 'https://arxiv.org/abs/2402.13753'}),\n", + " Document(page_content='Large language models (LLMs) have witnessed significant advancements, aiming to enhance their capabilities for interpreting and processing extensive textual data. LLMs like GPT-3 have revolutionized our interactions with AI, offering insights and analyses across various domains, from writing assistance to complex data interpretation. However, a key limitation has been their context window size ...', metadata={'id': '354ceab3-03ae-4141-94a1-4b0109182aab', 'source': 'https://www.marktechpost.com/2024/02/23/breaking-barriers-in-language-understanding-how-microsoft-ais-longrope-extends-large-language-models-to-a-2048k-token-context-window/'}),\n", + " Document(page_content='Large Language Models (LLMs) demonstrate significant capabilities but face challenges such as hallucination, outdated knowledge, and non-transparent, untraceable reasoning processes. Retrieval-Augmented Generation (RAG) has emerged as a promising solution by incorporating knowledge from external databases. This enhances the accuracy and credibility of the models, particularly for knowledge ...', metadata={'id': '85f7d7a1-2820-466c-ac80-975f4e2d65af', 'source': 'https://arxiv.org/abs/2312.10997'})]" + ] + }, + "execution_count": 41, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "retriever.invoke(\"What's a long context LLM anyway?\")" + ] + }, + { + "cell_type": "code", + "execution_count": 50, "metadata": {}, "outputs": [], "source": [ "section_writer_prompt = ChatPromptTemplate.from_messages(\n", " [\n", - " (\"system\", \"You are an expert Wikipedia writer. Complete your assigned WikiSection from the following outline:\\n\\n\"\n", - " \"{outline}\\n\\nCite your sources, using the following references:\\n\\n\\n{docs}\\n\"),\n", - " (\"user\", \"Write the full WikiSection for the {section} section.\")\n", + " (\n", + " \"system\",\n", + " \"You are an expert Wikipedia writer. Complete your assigned WikiSection from the following outline:\\n\\n\"\n", + " \"{outline}\\n\\nCite your sources, using the following references:\\n\\n\\n{docs}\\n\",\n", + " ),\n", + " (\"user\", \"Write the full WikiSection for the {section} section.\"),\n", " ]\n", ")\n", "\n", "\n", "async def retrieve(inputs: dict):\n", - " docs = await retriever.ainvoke(inputs['topic'] + \": \" + inputs[\"section\"])\n", - " formatted = \"\\n\".join([f'\\n{doc.page_content}\\n' for doc in docs])\n", - " return {\n", - " \"docs\": formatted,\n", - " **inputs\n", - " }\n", + " docs = await retriever.ainvoke(inputs[\"topic\"] + \": \" + inputs[\"section\"])\n", + " formatted = \"\\n\".join(\n", + " [\n", + " f'\\n{doc.page_content}\\n'\n", + " for doc in docs\n", + " ]\n", + " )\n", + " return {\"docs\": formatted, **inputs}\n", + "\n", "\n", "section_writer = (\n", " retrieve\n", " | section_writer_prompt\n", - " | ChatOpenAI(model=\"gpt-4-turbo-preview\").with_structured_output(WikiSection)\n", + " | long_context_llm.with_structured_output(WikiSection)\n", ")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Impact of million-plus token context window language models on RAG The Evolution of Large Context Windows in Language Models\n", + "## The Evolution of Large Context Windows in Language Models\n", + "\n", + "The evolution of large context windows in language models (LLMs) has been a critical factor in the advancement of natural language processing (NLP) technologies. Initially, LLMs were constrained by smaller context windows, limiting their understanding and generation capabilities. However, the demand for models capable of processing and integrating more extensive sequences of text has led to significant research and development efforts aimed at expanding these context windows.\n", + "\n", + "Over time, this push for larger context windows has seen the emergence of several key milestones that have progressively increased the amount of text LLMs can consider when generating responses or analyses. These milestones include models like Gemini 1.5, Mixtral, GPT-3.5-Turbo-16k, Llama2-7B-chat-4k, and LongRoPE, each contributing to the landscape of large context window LLMs in unique ways.\n", + "\n", + "Despite the benefits, expanding the context window size brings several challenges, including the high fine-tuning costs associated with processing longer sequences of text, the scarcity of long texts suitable for training these models, and the potential for catastrophic values introduced by new token positions in expanded contexts. Addressing these challenges has been central to the continued development and effectiveness of LLMs with large context windows.\n", + "\n", + "### Key Milestones\n", + "\n", + "The journey towards expanding the context window sizes of language models has been marked by several significant milestones. Gemini 1.5, Mixtral, GPT-3.5-Turbo-16k, Llama2-7B-chat-4k, and LongRoPE represent some of the most notable advancements in this area. Each of these models has pushed the boundaries of what was previously possible, setting new standards for the amount of text that can be processed and understood by LLMs.\n", + "\n", + "For instance, LongRoPE has made a groundbreaking contribution by extending the context window of pre-trained LLMs to an impressive 2048k tokens, far surpassing previous limits and opening up new possibilities for NLP applications.\n", + "\n", + "### Challenges Overcome\n", + "\n", + "Expanding the context window sizes of language models has not been without its challenges. High fine-tuning costs, the scarcity of suitable long texts for training, and the introduction of catastrophic values by new token positions have all posed significant hurdles.\n", + "\n", + "Technological and methodological advancements have been crucial in overcoming these challenges, allowing for the successful expansion of context windows beyond previous limitations. Innovations in model architecture, training methodologies, and data processing techniques have all played a role in addressing these issues and enabling the development of more capable and efficient LLMs.[0] https://arxiv.org/abs/2402.13753\n", + " [1] https://syncedreview.com/2024/02/25/microsofts-longrope-breaks-the-limit-of-context-window-of-llms-extents-it-to-2-million-tokens/\n", + " [2] https://www.marktechpost.com/2024/02/23/breaking-barriers-in-language-understanding-how-microsoft-ais-longrope-extends-large-language-models-to-a-2048k-token-context-window/\n" + ] + } + ], "source": [ - "section = await section_writer.abatch(\n", - " {\"outline\": refined_outline.as_str,\n", - " \"section\" : refined_outline.sections[1].section_title,\n", - " \"topic\": example_topic,\n", - "}\n", + "section = await section_writer.ainvoke(\n", + " {\n", + " \"outline\": refined_outline.as_str,\n", + " \"section\": refined_outline.sections[1].section_title,\n", + " \"topic\": example_topic,\n", + " }\n", ")\n", "print(section.as_str)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "## The Evolution of Large Context Windows in Language Models\n", + "\n", + "The evolution of large context windows in language models (LLMs) has been a critical factor in the advancement of natural language processing (NLP) technologies. Initially, LLMs were constrained by smaller context windows, limiting their understanding and generation capabilities. However, the demand for models capable of processing and integrating more extensive sequences of text has led to significant research and development efforts aimed at expanding these context windows.\n", + "\n", + "Over time, this push for larger context windows has seen the emergence of several key milestones that have progressively increased the amount of text LLMs can consider when generating responses or analyses. These milestones include models like Gemini 1.5, Mixtral, GPT-3.5-Turbo-16k, Llama2-7B-chat-4k, and LongRoPE, each contributing to the landscape of large context window LLMs in unique ways.\n", + "\n", + "Despite the benefits, expanding the context window size brings several challenges, including the high fine-tuning costs associated with processing longer sequences of text, the scarcity of long texts suitable for training these models, and the potential for catastrophic values introduced by new token positions in expanded contexts. Addressing these challenges has been central to the continued development and effectiveness of LLMs with large context windows.\n", + "\n", + "### Key Milestones\n", + "\n", + "The journey towards expanding the context window sizes of language models has been marked by several significant milestones. Gemini 1.5, Mixtral, GPT-3.5-Turbo-16k, Llama2-7B-chat-4k, and LongRoPE represent some of the most notable advancements in this area. Each of these models has pushed the boundaries of what was previously possible, setting new standards for the amount of text that can be processed and understood by LLMs.\n", + "\n", + "For instance, LongRoPE has made a groundbreaking contribution by extending the context window of pre-trained LLMs to an impressive 2048k tokens, far surpassing previous limits and opening up new possibilities for NLP applications.\n", + "\n", + "### Challenges Overcome\n", + "\n", + "Expanding the context window sizes of language models has not been without its challenges. High fine-tuning costs, the scarcity of suitable long texts for training, and the introduction of catastrophic values by new token positions have all posed significant hurdles.\n", + "\n", + "Technological and methodological advancements have been crucial in overcoming these challenges, allowing for the successful expansion of context windows beyond previous limitations. Innovations in model architecture, training methodologies, and data processing techniques have all played a role in addressing these issues and enabling the development of more capable and efficient LLMs.[0] https://arxiv.org/abs/2402.13753\n", + " [1] https://syncedreview.com/2024/02/25/microsofts-longrope-breaks-the-limit-of-context-window-of-llms-extents-it-to-2-million-tokens/\n", + " [2] https://www.marktechpost.com/2024/02/23/breaking-barriers-in-language-understanding-how-microsoft-ais-longrope-extends-large-language-models-to-a-2048k-token-context-window/\n" + ] + } + ], "source": [ "print(section.as_str)" ] @@ -918,27 +1128,96 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 54, "metadata": {}, "outputs": [], "source": [ "from langchain_core.output_parsers import StrOutputParser\n", + "\n", "writer_prompt = ChatPromptTemplate.from_messages(\n", - " [\n", - " (\"system\", \"You are an expert Wikipedia author. Write the complete wiki article on {topic} using the following section drafts:\\n\\n\"\n", - " \"{draft}\\n\\nStrictly follow Wikipedia format guidelines.\"),\n", - " (\"user\", 'Write the complete Wiki article using markdown format. Organize citations using footnotes like \"[1]\", avoiding duplicates in the footer.')\n", + " [\n", + " (\n", + " \"system\",\n", + " \"You are an expert Wikipedia author. Write the complete wiki article on {topic} using the following section drafts:\\n\\n\"\n", + " \"{draft}\\n\\nStrictly follow Wikipedia format guidelines.\",\n", + " ),\n", + " (\n", + " \"user\",\n", + " 'Write the complete Wiki article using markdown format. Organize citations using footnotes like \"[1]\", avoiding duplicates in the footer.',\n", + " ),\n", " ]\n", ")\n", "\n", - "writer = writer_prompt | ChatOpenAI(model=\"gpt-4-turbo-preview\") | StrOutputParser()" + "writer = writer_prompt | long_context_llm | StrOutputParser()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Impact of Million-Plus Token Context Window Language Models on RAG\n", + "\n", + "The development and implementation of million-plus token context window language models (LLMs) represent a significant milestone in the field of natural language processing (NLP). These models have dramatically enhanced the capabilities of retrieval-augmented generation (RAG) systems, enabling them to generate more accurate, contextually relevant, and nuanced text outputs. This article delves into the evolution of large context windows in LLMs, their impact on RAG, and the challenges faced along the way.\n", + "\n", + "## Contents\n", + "\n", + "1. [The Evolution of Large Context Windows in Language Models](#The-Evolution-of-Large-Context-Windows-in-Language-Models)\n", + " 1. [Key Milestones](#Key-Milestones)\n", + " 2. [Challenges Overcome](#Challenges-Overcome)\n", + "2. [Impact on Retrieval-Augmented Generation](#Impact-on-Retrieval-Augmented-Generation)\n", + " 1. [Enhanced Contextual Understanding](#Enhanced-Contextual-Understanding)\n", + " 2. [Improved Accuracy and Relevance](#Improved-Accuracy-and-Relevance)\n", + " 3. [Challenges and Solutions](#Challenges-and-Solutions)\n", + "3. [Conclusion](#Conclusion)\n", + "4. [References](#References)\n", + "\n", + "## The Evolution of Large Context Windows in Language Models\n", + "\n", + "The evolution of large context windows in language models (LLMs) has been instrumental in advancing natural language processing (NLP) technologies. Initially, these models were restricted by smaller context windows, limiting their comprehension and generation abilities. The push for models that could process and integrate longer text sequences led to significant research and development efforts aimed at expanding these context windows.\n", + "\n", + "### Key Milestones\n", + "\n", + "Several key milestones have marked the journey towards enlarging the context window sizes of language models, including Gemini 1.5, Mixtral, GPT-3.5-Turbo-16k, Llama2-7B-chat-4k, and LongRoPE. Each model has contributed uniquely to the landscape of large context window LLMs, progressively increasing the amount of text that LLMs can consider when generating responses or analyses. Notably, LongRoPE has been a groundbreaking advancement, extending the context window to an impressive 2048k tokens[1][2].\n", + "\n", + "### Challenges Overcome\n", + "\n", + "The expansion of context window sizes has encountered several challenges, such as high fine-tuning costs, the scarcity of suitable long texts for training, and the introduction of catastrophic values by new token positions. Technological and methodological advancements have been pivotal in overcoming these challenges, enabling the successful expansion of context windows[0].\n", + "\n", + "## Impact on Retrieval-Augmented Generation\n", + "\n", + "The implementation of million-plus token context window LLMs has had a profound impact on RAG systems, enhancing their performance in various ways.\n", + "\n", + "### Enhanced Contextual Understanding\n", + "\n", + "With the ability to process and integrate larger sequences of text, RAG systems can now generate responses that are more contextually relevant and nuanced. This has significantly improved the quality of text generation across diverse NLP applications.\n", + "\n", + "### Improved Accuracy and Relevance\n", + "\n", + "The expanded context windows allow RAG systems to retrieve and leverage more relevant information, leading to improvements in the accuracy and relevance of generated text. This capability is particularly beneficial in tasks that require a deep understanding of large text corpora, such as summarization and question-answering.\n", + "\n", + "### Challenges and Solutions\n", + "\n", + "Despite the benefits, integrating million-plus token context window LLMs into RAG systems presents challenges, including increased computational requirements and the need for more sophisticated retrieval mechanisms. Ongoing research and development are focused on addressing these issues, ensuring the continued advancement of RAG technologies.\n", + "\n", + "## Conclusion\n", + "\n", + "The advent of million-plus token context window LLMs has been a game-changer for RAG systems, significantly enhancing their text generation capabilities. Despite the challenges, the benefits of expanded context windows are undeniable, paving the way for more sophisticated and capable NLP applications.\n", + "\n", + "## References\n", + "\n", + "[0] \"Techniques for Expanding Context Window Sizes in Language Models,\" arXiv, 2024. https://arxiv.org/abs/2402.13753\n", + "\n", + "[1] \"Microsoft's LongRoPE Breaks the Limit of Context Window of LLMs, Extends it to 2 Million Tokens,\" SyncedReview, 2024. https://syncedreview.com/2024/02/25/microsofts-longrope-breaks-the-limit-of-context-window-of-llms-extents-it-to-2-million-tokens/\n", + "\n", + "[2] \"Breaking Barriers in Language Understanding: How Microsoft AI's LongRoPE Extends Large Language Models to a 2048k Token Context Window,\" MarkTechPost, 2024. https://www.marktechpost.com/2024/02/23/breaking-barriers-in-language-understanding-how-microsoft-ais-longrope-extends-large-language-models-to-a-2048k-token-context-window/" + ] + } + ], "source": [ "for tok in writer.stream({\"topic\": example_topic, \"draft\": section.as_str}):\n", " print(tok, end=\"\")" @@ -960,7 +1239,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 56, "metadata": {}, "outputs": [], "source": [ @@ -976,17 +1255,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 57, "metadata": {}, "outputs": [], "source": [ "import asyncio\n", "\n", + "\n", "async def initialize_research(state: ResearchState):\n", " topic = state[\"topic\"]\n", " coros = (\n", " generate_outline_direct.ainvoke({\"topic\": topic}),\n", - " survey_subjects.ainvoke(topic)\n", + " survey_subjects.ainvoke(topic),\n", " )\n", " results = await asyncio.gather(*coros)\n", " return {\n", @@ -995,15 +1275,24 @@ " \"editors\": results[1].editors,\n", " }\n", "\n", + "\n", "async def conduct_interviews(state: ResearchState):\n", " topic = state[\"topic\"]\n", - " initial_states = [{\n", - " \"editor\": editor,\n", - " \"messages\": [AIMessage(content=f\"So you said you were writing an article on {topic}?\", name=\"Subject Matter Expert\")]\n", - " } for editor in state[\"editors\"]]\n", + " initial_states = [\n", + " {\n", + " \"editor\": editor,\n", + " \"messages\": [\n", + " AIMessage(\n", + " content=f\"So you said you were writing an article on {topic}?\",\n", + " name=\"Subject Matter Expert\",\n", + " )\n", + " ],\n", + " }\n", + " for editor in state[\"editors\"]\n", + " ]\n", " # We call in to the sub-graph here\n", " interview_results = await interview_graph.abatch(initial_states)\n", - " \n", + "\n", " return {\n", " **state,\n", " \"interview_results\": interview_results,\n", @@ -1012,35 +1301,40 @@ "\n", "def format_conversation(interview_state):\n", " messages = interview_state[\"messages\"]\n", - " convo = \"\\n\".join(\n", - " f\"{m.name}: {m.content}\" for m in final_state[\"messages\"]\n", - " )\n", + " convo = \"\\n\".join(f\"{m.name}: {m.content}\" for m in final_state[\"messages\"])\n", " return f'Conversation with {interview_state[\"editor\"].name}\\n\\n' + convo\n", "\n", "\n", "async def refine_outline(state: ResearchState):\n", - " convos = \"\\n\\n\".join([format_conversation(interview_state) for interview_state in state[\"interview_results\"]])\n", - " \n", + " convos = \"\\n\\n\".join(\n", + " [\n", + " format_conversation(interview_state)\n", + " for interview_state in state[\"interview_results\"]\n", + " ]\n", + " )\n", + "\n", " updated_outline = await refine_outline_chain.ainvoke(\n", - " {\n", + " {\n", " \"topic\": state[\"topic\"],\n", " \"old_outline\": state[\"outline\"].as_str,\n", " \"conversations\": convos,\n", " }\n", " )\n", - " return {\n", - " **state,\n", - " \"outline\": updated_outline\n", - " }\n", + " return {**state, \"outline\": updated_outline}\n", + "\n", "\n", "async def index_references(state: ResearchState):\n", " all_docs = []\n", " for interview_state in state[\"interview_results\"]:\n", - " reference_docs = [Document(page_content=v, metadata={\"source\": k}) for k, v in interview_state[\"references\"].items()]\n", + " reference_docs = [\n", + " Document(page_content=v, metadata={\"source\": k})\n", + " for k, v in interview_state[\"references\"].items()\n", + " ]\n", " all_docs.extend(reference_docs)\n", " await vectorstore.aadd_documents(all_docs)\n", " return state\n", "\n", + "\n", "async def write_sections(state: ResearchState):\n", " outline = state[\"outline\"]\n", " sections = await section_writer.abatch(\n", @@ -1058,11 +1352,12 @@ " \"sections\": sections,\n", " }\n", "\n", + "\n", "async def write_article(state: ResearchState):\n", " topic = state[\"topic\"]\n", " sections = state[\"sections\"]\n", " draft = \"\\n\\n\".join([section.as_str for section in sections])\n", - " article = writer.ainvoke({\"topic\": example_topic, \"draft\": draft})\n", + " article = await writer.ainvoke({\"topic\": example_topic, \"draft\": draft})\n", " return {\n", " **state,\n", " \"article\": article,\n", @@ -1071,7 +1366,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 58, "metadata": {}, "outputs": [], "source": [ @@ -1090,7 +1385,7 @@ " name, node = nodes[i]\n", " builder_of_storm.add_node(name, node)\n", " if i > 0:\n", - " builder_of_storm.add_edge(nodes[i-1][0], name)\n", + " builder_of_storm.add_edge(nodes[i - 1][0], name)\n", "\n", "builder_of_storm.set_entry_point(nodes[0][0])\n", "builder_of_storm.set_finish_point(nodes[-1][0])\n", @@ -1099,13 +1394,44 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 59, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "init_research\n", + "-- {'topic': 'NVIDIA 2024 Q1 earnings report', 'outline': Outline(page_title='NVIDIA 2024 Q1 Earnings Report', sections=[Section(section_title='Overview', description='Brief introduction to NVIDIA and an overview of the 2024 Q1 earnings report', subsections=None), Section(section_title='Financial Perfo\n", + "conduct_interviews\n", + "-- {'topic': 'NVIDIA 2024 Q1 earnings report', 'outline': Outline(page_title='NVIDIA 2024 Q1 Earnings Report', sections=[Section(section_title='Overview', description='Brief introduction to NVIDIA and an overview of the 2024 Q1 earnings report', subsections=None), Section(section_title='Financial Perfo\n", + "refine_outline\n", + "-- {'topic': 'NVIDIA 2024 Q1 earnings report', 'outline': Outline(page_title='Impact of Million-Plus Token Context Window Language Models on RAG', sections=[Section(section_title='Introduction', description='An overview of the article, including the significance of million-plus token context window lan\n", + "index_references\n", + "-- {'topic': 'NVIDIA 2024 Q1 earnings report', 'outline': Outline(page_title='Impact of Million-Plus Token Context Window Language Models on RAG', sections=[Section(section_title='Introduction', description='An overview of the article, including the significance of million-plus token context window lan\n", + "NVIDIA 2024 Q1 earnings report Introduction\n", + "NVIDIA 2024 Q1 earnings report Background\n", + "NVIDIA 2024 Q1 earnings report Impact of Large Context Windows on RAG\n", + "NVIDIA 2024 Q1 earnings report Case Studies\n", + "NVIDIA 2024 Q1 earnings report RAG's Use of External Knowledge Sources\n", + "NVIDIA 2024 Q1 earnings report Future Outlook\n", + "NVIDIA 2024 Q1 earnings report Conclusions\n", + "NVIDIA 2024 Q1 earnings report References\n", + "write_sections\n", + "-- {'topic': 'NVIDIA 2024 Q1 earnings report', 'outline': Outline(page_title='Impact of Million-Plus Token Context Window Language Models on RAG', sections=[Section(section_title='Introduction', description='An overview of the article, including the significance of million-plus token context window lan\n", + "write_article\n", + "-- {'topic': 'NVIDIA 2024 Q1 earnings report', 'outline': Outline(page_title='Impact of Million-Plus Token Context Window Language Models on RAG', sections=[Section(section_title='Introduction', description='An overview of the article, including the significance of million-plus token context window lan\n", + "__end__\n", + "-- {'topic': 'NVIDIA 2024 Q1 earnings report', 'outline': Outline(page_title='Impact of Million-Plus Token Context Window Language Models on RAG', sections=[Section(section_title='Introduction', description='An overview of the article, including the significance of million-plus token context window lan\n" + ] + } + ], "source": [ - "async for step in storm.astream({\n", + "async for step in storm.astream(\n", + " {\n", " \"topic\": \"NVIDIA 2024 Q1 earnings report\",\n", - " }):\n", + " }\n", + "):\n", " name = next(iter(step))\n", " print(name)\n", " print(\"-- \", str(step[name])[:300])\n", @@ -1113,6 +1439,83 @@ " results = step" ] }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [], + "source": [ + "article = results[END][\"article\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# Impact of Million-Plus Token Context Window Language Models on RAG\n", + "\n", + "The **Impact of Million-Plus Token Context Window Language Models on Retrieval-Augmented Generation (RAG)** reflects a significant advancement in the field of natural language processing (NLP) and artificial intelligence (AI). This development has broadened the capabilities of AI systems in understanding and generating human-like text by integrating large-scale language models with external knowledge sources to produce contextually rich responses.\n", + "\n", + "## Contents\n", + "\n", + "- [Introduction](#Introduction)\n", + "- [Background](#Background)\n", + "- [Impact of Million-Plus Token Context Window Language Models on RAG](#Impact-of-Million-Plus-Token-Context-Window-Language-Models-on-RAG)\n", + " - [The Evolution of Large Context Windows in Language Models](#The-Evolution-of-Large-Context-Windows-in-Language-Models)\n", + " - [Integration Challenges with RAG](#Integration-Challenges-with-RAG)\n", + " - [Enhancing RAG with External Knowledge Sources](#Enhancing-RAG-with-External-Knowledge-Sources)\n", + " - [Applications and Future Directions](#Applications-and-Future-Directions)\n", + "- [Case Studies](#Case-Studies)\n", + "- [Conclusions](#Conclusions)\n", + "- [References](#References)\n", + "\n", + "## Introduction\n", + "\n", + "The advent of million-plus token context window language models marks a significant milestone in NLP and AI. These models process and understand vast amounts of text, enhancing machine learning applications, particularly in text generation and comprehension. The integration within the RAG framework improves accuracy, relevancy, and contextual richness of responses, addressing limitations of traditional language models.\n", + "\n", + "## Background\n", + "\n", + "The development of large language models (LLMs) with increasing parameter sizes and context windows has enhanced their capability to generate coherent and contextually relevant text outputs. The expansion of the context window has been crucial for understanding and generating complex texts. RAG, by integrating external knowledge sources, allows LLMs to provide more accurate and updated responses, opening new possibilities in NLP.\n", + "\n", + "## Impact of Million-Plus Token Context Window Language Models on RAG\n", + "\n", + "### The Evolution of Large Context Windows in Language Models\n", + "Models like Gemini 1.5, Mixtral, and GPT-3.5-Turbo-16k have pushed technical boundaries, addressing challenges in increasing context window size, enhancing understanding and text generation capabilities.\n", + "\n", + "### Integration Challenges with RAG\n", + "Integrating these models into RAG frameworks introduces challenges such as high fine-tuning costs and managing catastrophic values. Technological innovations and approaches to mitigate these effects are crucial for successful integration.\n", + "\n", + "### Enhancing RAG with External Knowledge Sources\n", + "The integration benefits RAG by addressing challenges like hallucination and outdated knowledge, leveraging up-to-date information to enhance response accuracy and credibility.\n", + "\n", + "### Applications and Future Directions\n", + "This integration opens up applications in chatbots, content creation, and information retrieval. Future developments aim to improve these models' efficiency, accuracy, and application breadth.\n", + "\n", + "## Case Studies\n", + "\n", + "Specific case studies demonstrate practical applications and challenges in integrating advanced language models with RAG, highlighting improvements in retrieval capabilities, data efficiency, and addressing information overload.\n", + "\n", + "## Conclusions\n", + "\n", + "The integration of million-plus token context window language models with RAG has significantly advanced NLP capabilities, enhancing machine understanding and generation of human language. Despite challenges, technological and strategic advancements continue to push the boundaries of what is possible in NLP, promising more sophisticated and nuanced AI-driven technologies.\n", + "\n", + "## References\n", + "\n", + "1. NVIDIA Announces Financial Results for First Quarter Fiscal 2024. [NVIDIA News](https://nvidianews.nvidia.com/news/nvidia-announces-financial-results-for-first-quarter-fiscal-2024)\n", + "\n", + "2. Nvidia Q1 Earnings Report. [InvestorPlace](https://investorplace.com/market360/2023/05/nvidia-q1-earnings-report/)\n" + ] + } + ], + "source": [ + "print(article)" + ] + }, { "cell_type": "code", "execution_count": null,