diff --git a/docs/_scripts/generate_llms_text.py b/docs/_scripts/generate_llms_text.py
index d5485f2e3..d71f28589 100644
--- a/docs/_scripts/generate_llms_text.py
+++ b/docs/_scripts/generate_llms_text.py
@@ -1,10 +1,16 @@
"""Experimental script to generate consolidated llms text from the docs."""
+import asyncio
import glob
import os
-from typing import TypedDict, List
+from typing import TypedDict, List, Optional
+import pydantic
+import re
+from pydantic import BaseModel, Field
+from langchain_core.rate_limiters import InMemoryRateLimiter
import yaml
+from langchain.chat_models import init_chat_model
from mkdocs.structure.files import File
from mkdocs.structure.pages import Page
from yaml import SafeLoader
@@ -16,7 +22,49 @@ HERE = os.path.dirname(os.path.abspath(__file__))
SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(HERE), "docs"))
-def generate_full_llms_text(output_file: str) -> str:
+async def convert_ipynb_to_md(file_path: str) -> Optional[str]:
+ """Process a file (markdown or notebook) to markdown format.
+
+ Args:
+ file_path: Path to the file to process
+
+ Returns:
+ Processed markdown content if successful, None otherwise
+ """
+ rel_path = os.path.relpath(file_path, SOURCE_DIR)
+
+ # Create File and Page objects to match mkdocs structure
+ file_obj = File(
+ path=rel_path, src_dir=SOURCE_DIR, dest_dir="", use_directory_urls=True
+ )
+ page = Page(
+ title="",
+ file=file_obj,
+ config={},
+ )
+
+ try:
+ # Read raw content
+ with open(file_path, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ # Convert to markdown without logic to resolve API references
+ processed_content = _on_page_markdown_with_config(
+ content, page, add_api_references=False, remove_base64_images=True
+ )
+ # Remove self-closing img tags ![]()
+ processed_content = re.sub(r"]*/>", "", processed_content)
+ # Remove img tags with content ![]()
...
+ processed_content = re.sub(
+ r"]*>.*?", "", processed_content, flags=re.DOTALL
+ )
+ return processed_content
+ except Exception as e:
+ print(f"Error processing file {file_path}: {e}")
+ return None
+
+
+async def generate_full_llms_text(output_file: str) -> None:
"""Generate a consolidated text file from markdown/notebook files for LLM training.
Args:
@@ -24,11 +72,9 @@ def generate_full_llms_text(output_file: str) -> str:
"""
# Collect all markdown and notebook files
all_files = glob.glob(os.path.join(SOURCE_DIR, "how-tos/*.md"), recursive=True)
-
all_files.extend(
glob.glob(os.path.join(SOURCE_DIR, "how-tos/*.ipynb"), recursive=True)
)
- # Add all concepts
all_files.extend(
glob.glob(os.path.join(SOURCE_DIR, "concepts/*.md"), recursive=True)
)
@@ -38,30 +84,14 @@ def generate_full_llms_text(output_file: str) -> str:
all_content = []
- # Process each file
- for file_path in all_files:
- print(f"Processing {file_path}")
- rel_path = os.path.relpath(file_path, SOURCE_DIR)
+ # Process files concurrently
+ tasks = [convert_ipynb_to_md(file_path) for file_path in all_files]
+ results = await asyncio.gather(*tasks)
- # Create File and Page objects to match mkdocs structure
- file_obj = File(
- path=rel_path, src_dir=SOURCE_DIR, dest_dir="", use_directory_urls=True
- )
- page = Page(
- title="",
- file=file_obj,
- config={},
- )
-
- # Read raw content
- with open(file_path, "r", encoding="utf-8") as f:
- content = f.read()
-
- # Convert to markdown without logic to resolve API references
- processed_content = _on_page_markdown_with_config(
- content, page, add_api_references=False, remove_base64_images=True
- )
+ # Combine results with file paths
+ for file_path, processed_content in zip(all_files, results):
if processed_content:
+ rel_path = os.path.relpath(file_path, SOURCE_DIR)
# Add file name
all_content.append(f"---\n{rel_path}\n---")
# Add content
@@ -86,6 +116,7 @@ class NavItem(TypedDict):
title: str
url: str
hierarchy: tuple[str, ...]
+ description: str
def _flatten_nav(
@@ -98,7 +129,14 @@ def _flatten_nav(
new_path = path + (title,)
if isinstance(node, str):
# Leaf page
- flat.append({"title": title, "url": node, "hierarchy": new_path})
+ flat.append(
+ {
+ "title": title,
+ "url": node,
+ "hierarchy": new_path,
+ "description": "",
+ }
+ )
elif isinstance(node, list):
# Dive in, carrying along the updated path
flat.extend(_flatten_nav(node, new_path))
@@ -109,14 +147,82 @@ def _flatten_nav(
elif isinstance(item, str):
# Bare string entry → use itself as title, and as URL
new_path = path + (item,)
- flat.append({"title": item, "url": item, "hierarchy": new_path})
+ flat.append(
+ {"title": item, "url": item, "hierarchy": new_path, "description": ""}
+ )
else:
raise TypeError(f"Unexpected item type {type(item)} in nav")
return flat
-def generate_nav_links_text(output_file: str, *, replace_links: bool = False) -> None:
- """Generate a text file containing navigation structure and links from mkdocs.yaml."""
+class PageInfo(BaseModel):
+ title: str = Field(description="The title of the page")
+ description: str = Field(
+ description="A short description of the page no longer than 3 sentences "
+ "explaining the kind of content that can be found in the page."
+ )
+
+
+async def process_nav_items(nav_items: list[NavItem]) -> list[NavItem]:
+ """Open the contents of each nav item and come up with a better title and description."""
+ rate_limiter = InMemoryRateLimiter(requests_per_second=10)
+ model = init_chat_model("gpt-4o-mini", temperature=0.0, rate_limiter=rate_limiter)
+ model = model.with_structured_output(PageInfo)
+
+ async def process_single_item(item: NavItem) -> NavItem:
+ path = item["url"]
+ file_path = os.path.join(SOURCE_DIR, path)
+
+ # Process the file content (handles both markdown and notebooks)
+ if path.endswith(".ipynb"):
+ content = await convert_ipynb_to_md(file_path)
+ else:
+ with open(file_path, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ if not content:
+ return item
+
+ # Generate a better title and description
+ response = await model.ainvoke(
+ [
+ {
+ "role": "system",
+ "content": "You are a technical documentation writer. "
+ "You are given a markdown page of documentation. "
+ "Please come up with an appropriate title and "
+ "description for the page. The description should "
+ "be a short summary of the page content that is "
+ "no longer than 3 sentences.",
+ },
+ {
+ "role": "user",
+ "content": "The markdown page is as follows:\n\n" + content,
+ },
+ ]
+ )
+ return {
+ "title": response.title,
+ "url": item["url"],
+ "hierarchy": item["hierarchy"],
+ "description": response.description,
+ }
+
+ # Remove any items that start with http:// or https:// looking only for
+ # local file at this stages.
+ nav_items = [
+ item for item in nav_items if not item["url"].startswith(("http://", "https://"))
+ ]
+ # Process items in parallel
+ tasks = [process_single_item(item) for item in nav_items]
+ new_nav_items = await asyncio.gather(*tasks)
+ return new_nav_items
+
+
+async def generate_nav_links_text(
+ output_file: str, *, replace_links: bool = False
+) -> None:
+ """Generate llms.txt from mkdocs.yaml."""
# Get path to mkdocs.yaml relative to this script
script_dir = os.path.dirname(os.path.abspath(__file__))
mkdocs_path = os.path.join(os.path.dirname(script_dir), "mkdocs.yml")
@@ -129,15 +235,15 @@ def generate_nav_links_text(output_file: str, *, replace_links: bool = False) ->
nav = config.get("nav", [])
flattened = _flatten_nav(nav)
+ processed_nav = await process_nav_items(flattened)
+
with open(output_file, "w") as f:
current_section = None
- for item in flattened:
+ for item in processed_nav:
# Get the top-level section (first item in hierarchy)
section = item["hierarchy"][0]
- if section not in {
- "Guides", "Examples", "Resources"
- }:
+ if section not in {"Guides", "Examples", "Resources"}:
continue
# If we're starting a new section, add a heading
@@ -145,15 +251,7 @@ def generate_nav_links_text(output_file: str, *, replace_links: bool = False) ->
f.write(f"\n# {section}\n\n")
current_section = section
- # Add the item as a bullet point with title and link
- # Include full hierarchy path in title, separated by " > "
- hierarchy_path = " > ".join(item["hierarchy"][1:])
- title = (
- f"{item['title']} ({hierarchy_path})"
- if hierarchy_path
- else item["title"]
- )
-
+ title = item["title"]
# Process URL based on replace_links flag
url = item["url"]
if replace_links:
@@ -163,7 +261,7 @@ def generate_nav_links_text(output_file: str, *, replace_links: bool = False) ->
url = url.rstrip("/") + "/"
url = f"https://langchain-ai.github.io/langgraph/{url}"
- f.write(f"- [{title}]({url})\n")
+ f.write(f"- [{title}]({url}): {item['description']}\n")
if __name__ == "__main__":
@@ -188,6 +286,10 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.link_only:
- generate_nav_links_text(args.output_file, replace_links=args.replace_links)
+ coro = generate_nav_links_text(
+ args.output_file, replace_links=args.replace_links
+ )
else:
- generate_full_llms_text(args.output_file)
+ coro = generate_full_llms_text(args.output_file)
+
+ asyncio.run(coro)