WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-08-29 11:19:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

docs: improve llms.txt to generate title and description based on content (#4775)

Browse Source 
Update llms.txt based on content
This commit is contained in:
Eugene Yurtsev
2025-05-21 14:39:13 -04:00
committed by GitHub
parent b8daf323ac
commit fd0b70eb05
1 changed files with 148 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/_scripts/generate_llms_text.py
+148 -46
View File
@@ -1,10 +1,16 @@
"""Experimental script to generate consolidated llms text from the docs."""
import asyncio
import glob
import os
from typing import TypedDict, List
from typing import TypedDict, List, Optional
import pydantic
import re
from pydantic import BaseModel, Field
from langchain_core.rate_limiters import InMemoryRateLimiter
import yaml
from langchain.chat_models import init_chat_model
from mkdocs.structure.files import File
from mkdocs.structure.pages import Page
from yaml import SafeLoader
@@ -16,7 +22,49 @@ HERE = os.path.dirname(os.path.abspath(__file__))
SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(HERE), "docs"))
def generate_full_llms_text(output_file: str) -> str:
async def convert_ipynb_to_md(file_path: str) -> Optional[str]:
"""Process a file (markdown or notebook) to markdown format.
Args:
file_path: Path to the file to process
Returns:
Processed markdown content if successful, None otherwise
"""
rel_path = os.path.relpath(file_path, SOURCE_DIR)
# Create File and Page objects to match mkdocs structure
file_obj = File(
path=rel_path, src_dir=SOURCE_DIR, dest_dir="", use_directory_urls=True
)
page = Page(
title="",
file=file_obj,
config={},
)
try:
# Read raw content
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
# Convert to markdown without logic to resolve API references
processed_content = _on_page_markdown_with_config(
content, page, add_api_references=False, remove_base64_images=True
)
# Remove self-closing img tags <img ... />
processed_content = re.sub(r"<img[^>]*/>", "", processed_content)
# Remove img tags with content <img ...>...</img>
processed_content = re.sub(
r"<img[^>]*>.*?</img>", "", processed_content, flags=re.DOTALL
)
return processed_content
except Exception as e:
print(f"Error processing file {file_path}: {e}")
return None
async def generate_full_llms_text(output_file: str) -> None:
"""Generate a consolidated text file from markdown/notebook files for LLM training.
Args:
@@ -24,11 +72,9 @@ def generate_full_llms_text(output_file: str) -> str:
"""
# Collect all markdown and notebook files
all_files = glob.glob(os.path.join(SOURCE_DIR, "how-tos/*.md"), recursive=True)
all_files.extend(
glob.glob(os.path.join(SOURCE_DIR, "how-tos/*.ipynb"), recursive=True)
)
# Add all concepts
all_files.extend(
glob.glob(os.path.join(SOURCE_DIR, "concepts/*.md"), recursive=True)
)
@@ -38,30 +84,14 @@ def generate_full_llms_text(output_file: str) -> str:
all_content = []
# Process each file
for file_path in all_files:
print(f"Processing {file_path}")
rel_path = os.path.relpath(file_path, SOURCE_DIR)
# Process files concurrently
tasks = [convert_ipynb_to_md(file_path) for file_path in all_files]
results = await asyncio.gather(*tasks)
# Create File and Page objects to match mkdocs structure
file_obj = File(
path=rel_path, src_dir=SOURCE_DIR, dest_dir="", use_directory_urls=True
)
page = Page(
title="",
file=file_obj,
config={},
)
# Read raw content
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
# Convert to markdown without logic to resolve API references
processed_content = _on_page_markdown_with_config(
content, page, add_api_references=False, remove_base64_images=True
)
# Combine results with file paths
for file_path, processed_content in zip(all_files, results):
if processed_content:
rel_path = os.path.relpath(file_path, SOURCE_DIR)
# Add file name
all_content.append(f"---\n{rel_path}\n---")
# Add content
@@ -86,6 +116,7 @@ class NavItem(TypedDict):
title: str
url: str
hierarchy: tuple[str, ...]
description: str
def _flatten_nav(
@@ -98,7 +129,14 @@ def _flatten_nav(
new_path = path + (title,)
if isinstance(node, str):
# Leaf page
flat.append({"title": title, "url": node, "hierarchy": new_path})
flat.append(
{
"title": title,
"url": node,
"hierarchy": new_path,
"description": "",
}
)
elif isinstance(node, list):
# Dive in, carrying along the updated path
flat.extend(_flatten_nav(node, new_path))
@@ -109,14 +147,82 @@ def _flatten_nav(
elif isinstance(item, str):
# Bare string entry → use itself as title, and as URL
new_path = path + (item,)
flat.append({"title": item, "url": item, "hierarchy": new_path})
flat.append(
{"title": item, "url": item, "hierarchy": new_path, "description": ""}
)
else:
raise TypeError(f"Unexpected item type {type(item)} in nav")
return flat
def generate_nav_links_text(output_file: str, *, replace_links: bool = False) -> None:
"""Generate a text file containing navigation structure and links from mkdocs.yaml."""
class PageInfo(BaseModel):
title: str = Field(description="The title of the page")
description: str = Field(
description="A short description of the page no longer than 3 sentences "
"explaining the kind of content that can be found in the page."
)
async def process_nav_items(nav_items: list[NavItem]) -> list[NavItem]:
"""Open the contents of each nav item and come up with a better title and description."""
rate_limiter = InMemoryRateLimiter(requests_per_second=10)
model = init_chat_model("gpt-4o-mini", temperature=0.0, rate_limiter=rate_limiter)
model = model.with_structured_output(PageInfo)
async def process_single_item(item: NavItem) -> NavItem:
path = item["url"]
file_path = os.path.join(SOURCE_DIR, path)
# Process the file content (handles both markdown and notebooks)
if path.endswith(".ipynb"):
content = await convert_ipynb_to_md(file_path)
else:
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
if not content:
return item
# Generate a better title and description
response = await model.ainvoke(
[
{
"role": "system",
"content": "You are a technical documentation writer. "
"You are given a markdown page of documentation. "
"Please come up with an appropriate title and "
"description for the page. The description should "
"be a short summary of the page content that is "
"no longer than 3 sentences.",
},
{
"role": "user",
"content": "The markdown page is as follows:\n\n" + content,
},
]
)
return {
"title": response.title,
"url": item["url"],
"hierarchy": item["hierarchy"],
"description": response.description,
}
# Remove any items that start with http:// or https:// looking only for
# local file at this stages.
nav_items = [
item for item in nav_items if not item["url"].startswith(("http://", "https://"))
]
# Process items in parallel
tasks = [process_single_item(item) for item in nav_items]
new_nav_items = await asyncio.gather(*tasks)
return new_nav_items
async def generate_nav_links_text(
output_file: str, *, replace_links: bool = False
) -> None:
"""Generate llms.txt from mkdocs.yaml."""
# Get path to mkdocs.yaml relative to this script
script_dir = os.path.dirname(os.path.abspath(__file__))
mkdocs_path = os.path.join(os.path.dirname(script_dir), "mkdocs.yml")
@@ -129,15 +235,15 @@ def generate_nav_links_text(output_file: str, *, replace_links: bool = False) ->
nav = config.get("nav", [])
flattened = _flatten_nav(nav)
processed_nav = await process_nav_items(flattened)
with open(output_file, "w") as f:
current_section = None
for item in flattened:
for item in processed_nav:
# Get the top-level section (first item in hierarchy)
section = item["hierarchy"][0]
if section not in {
"Guides", "Examples", "Resources"
}:
if section not in {"Guides", "Examples", "Resources"}:
continue
# If we're starting a new section, add a heading
@@ -145,15 +251,7 @@ def generate_nav_links_text(output_file: str, *, replace_links: bool = False) ->
f.write(f"\n# {section}\n\n")
current_section = section
# Add the item as a bullet point with title and link
# Include full hierarchy path in title, separated by " > "
hierarchy_path = " > ".join(item["hierarchy"][1:])
title = (
f"{item['title']} ({hierarchy_path})"
if hierarchy_path
else item["title"]
)
title = item["title"]
# Process URL based on replace_links flag
url = item["url"]
if replace_links:
@@ -163,7 +261,7 @@ def generate_nav_links_text(output_file: str, *, replace_links: bool = False) ->
url = url.rstrip("/") + "/"
url = f"https://langchain-ai.github.io/langgraph/{url}"
f.write(f"- [{title}]({url})\n")
f.write(f"- [{title}]({url}): {item['description']}\n")
if __name__ == "__main__":
@@ -188,6 +286,10 @@ if __name__ == "__main__":
args = parser.parse_args()
if args.link_only:
generate_nav_links_text(args.output_file, replace_links=args.replace_links)
coro = generate_nav_links_text(
args.output_file, replace_links=args.replace_links
)
else:
generate_full_llms_text(args.output_file)
coro = generate_full_llms_text(args.output_file)
asyncio.run(coro)