From f59a1339c930601cf59f3c560f0664e76c20c45d Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Fri, 11 Jul 2025 16:44:46 -0400 Subject: [PATCH] chore(docs): Consolidate hooks for copy markdown and notebooks (#5459) Consolidating the hooks to avoid duplication of logic We need this change for consolidating js and python content: we need include-markdown to run as a mkdocs plugin before our pipeline (rather than as markdown extension which runs after our hooks plugin). --- docs/_scripts/copy_page_hooks.py | 162 - docs/_scripts/notebook_hooks.py | 56 +- docs/docs/agents/models.md | 2 +- docs/docs/how-tos/graph-api.md | 2 +- .../docs/how-tos/memory/semantic-search.ipynb | 2 +- docs/docs/index.md | 2 +- docs/docs/snippets/chat_model_tabs.md | 87 + .../get-started/1-build-basic-chatbot.md | 2 +- .../docs/tutorials/get-started/2-add-tools.md | 4 +- .../tutorials/get-started/3-add-memory.md | 2 +- .../get-started/4-human-in-the-loop.md | 4 +- .../get-started/5-customize-state.md | 2 +- .../tutorials/get-started/6-time-travel.md | 2 +- .../tutorials/llm-compiler/LLMCompiler.ipynb | 16 +- .../plan-and-execute/plan-and-execute.ipynb | 2 +- docs/docs/tutorials/tnt-llm/tnt-llm.ipynb | 14 +- docs/mkdocs.yml | 4 +- docs/pyproject.toml | 3 +- docs/uv.lock | 3552 +++++++---------- 19 files changed, 1629 insertions(+), 2291 deletions(-) delete mode 100644 docs/_scripts/copy_page_hooks.py create mode 100644 docs/docs/snippets/chat_model_tabs.md diff --git a/docs/_scripts/copy_page_hooks.py b/docs/_scripts/copy_page_hooks.py deleted file mode 100644 index 2dd42b29c..000000000 --- a/docs/_scripts/copy_page_hooks.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -Copy page functionality hooks for MkDocs. - -This module provides hooks to inject original markdown content into HTML pages -for the copy page functionality, allowing users to copy clean markdown content -optimized for LLMs. -""" - -import json -import re -from pathlib import Path -from typing import Optional - -from mkdocs.config.defaults import MkDocsConfig -from mkdocs.structure.pages import Page - - -def _process_includes(content: str, docs_dir: Path) -> str: - """Process MkDocs includes like {!../README.md!}.""" - include_pattern = r'\{!([^!]+)!\}' - - def replace_include(match): - include_path = match.group(1) - # Resolve relative path - if include_path.startswith('../'): - # Go up from docs dir - include_file = docs_dir.parent / include_path[3:] - else: - include_file = docs_dir / include_path - - try: - with open(include_file, 'r', encoding='utf-8') as f: - included_content = f.read() - # Remove frontmatter from included content to avoid duplication - included_content = re.sub(r'^---\n.*?\n---\n', '', included_content, flags=re.DOTALL) - return included_content - except: - return f"[Content from {include_path}]" - - return re.sub(include_pattern, replace_include, content) - - -def _clean_markdown(content: str) -> str: - """Minimal cleanup of markdown content - preserve original as much as possible.""" - # Remove frontmatter - content = re.sub(r'^---\n.*?\n---\n', '', content, flags=re.DOTALL) - - # Remove script tags (security) - content = re.sub(r']*>.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - - # Remove style tags (security) - content = re.sub(r']*>.*?', '', content, flags=re.DOTALL | re.IGNORECASE) - - # Remove HTML comments - content = re.sub(r'', '', content, flags=re.DOTALL) - - # Just strip and return - preserve original structure - return content.strip() - - -def inject_markdown_content(html: str, page: Page, config: MkDocsConfig) -> str: - """ - Inject the original markdown content into the HTML for copy page functionality. - - Args: - html: The HTML content to inject into - page: The MkDocs page object - config: The MkDocs configuration - - Returns: - Modified HTML with markdown content injected as JSON - """ - if not hasattr(page, 'file') or not page.file: - return html - - # Get the original markdown file path - docs_dir = Path(config.get('docs_dir', 'docs')) - src_path = page.file.src_path - - # Handle different file types - if src_path.endswith('.ipynb'): - # For notebook files, we might want to use the converted markdown - # For now, just return the HTML as-is - return html - - markdown_file = docs_dir / src_path - - if not markdown_file.exists(): - return html - - try: - # Read the original markdown content - with open(markdown_file, 'r', encoding='utf-8') as f: - markdown_content = f.read() - - # Special handling for index page - use relative path to the actual README.md - if src_path == 'index.md': - # Relative path to the repository README.md file (go up two levels from docs/docs) - readme_path = docs_dir.parent.parent / 'README.md' - - try: - with open(readme_path, 'r', encoding='utf-8') as f: - readme_content = f.read() - # Remove frontmatter if present - processed_markdown = re.sub(r'^---\n.*?\n---\n', '', readme_content, flags=re.DOTALL) - processed_markdown = processed_markdown.strip() - except Exception as e: - # If we can't read the README, fallback to original behavior - processed_markdown = _process_includes(markdown_content, docs_dir) - processed_markdown = re.sub(r'^---\n.*?\n---\n', '', processed_markdown, flags=re.DOTALL) - processed_markdown = processed_markdown.strip() - else: - # Process any includes in the markdown to get the full content - processed_markdown = _process_includes(markdown_content, docs_dir) - # Clean up the processed markdown normally for other pages - processed_markdown = _clean_markdown(processed_markdown) - - # Create the JSON data - markdown_data = { - 'markdown': processed_markdown, - 'title': page.title or 'Page Content', - 'url': page.url or '' - } - - # Properly escape the JSON for HTML - json_content = json.dumps(markdown_data, ensure_ascii=False) - json_content = json_content.replace('{json_content}' - - # Insert before if it exists, otherwise before - if '' in html: - html = html.replace('', f'{script_content}') - elif '' in html: - html = html.replace('', f'{script_content}') - - except Exception as e: - # If anything goes wrong, just return the original HTML - # Could log the error here if needed - pass - - return html - - -def on_post_page(output: str, page: Page, config: MkDocsConfig) -> str: - """ - MkDocs hook to inject markdown content into HTML pages. - - This hook is called after each page is rendered and injects the original - markdown content as JSON for the copy page functionality. - - Args: - output: The HTML output of the page - page: The MkDocs page object - config: The MkDocs configuration - - Returns: - Modified HTML with markdown content injected - """ - return inject_markdown_content(output, page, config) \ No newline at end of file diff --git a/docs/_scripts/notebook_hooks.py b/docs/_scripts/notebook_hooks.py index db31e41ce..5ddb38289 100644 --- a/docs/_scripts/notebook_hooks.py +++ b/docs/_scripts/notebook_hooks.py @@ -3,6 +3,7 @@ Lifecycle events: https://www.mkdocs.org/dev-guide/plugins/#events """ +import json import logging import os import posixpath @@ -15,8 +16,8 @@ from mkdocs.structure.files import Files, File from mkdocs.structure.pages import Page from _scripts.generate_api_reference_links import update_markdown_with_imports -from _scripts.notebook_convert import convert_notebook from _scripts.link_map import JS_LINK_MAP +from _scripts.notebook_convert import convert_notebook logger = logging.getLogger(__name__) logging.basicConfig() @@ -356,12 +357,16 @@ def _on_page_markdown_with_config( def on_page_markdown(markdown: str, page: Page, **kwargs: Dict[str, Any]): - return _on_page_markdown_with_config( - markdown, - page, - add_api_references=True, - **kwargs, + finalized_markdown = ( + _on_page_markdown_with_config( + markdown, + page, + add_api_references=True, + **kwargs, + ) ) + page.meta["original_markdown"] = finalized_markdown + return finalized_markdown # redirects @@ -431,20 +436,51 @@ height="0" width="0" style="display:none;visibility:hidden"> else: return html # fallback if no found +def _inject_markdown_into_html(html: str, page: Page) -> str: + """Inject the original markdown content into the HTML page as JSON.""" + original_markdown = page.meta.get("original_markdown", "") + if not original_markdown: + return html + markdown_data = { + "markdown": original_markdown, + "title": page.title or "Page Content", + "url": page.url or "", + } -def on_post_page(output: str, page: Page, config: MkDocsConfig) -> str: + # Properly escape the JSON for HTML + json_content = json.dumps(markdown_data, ensure_ascii=False) + + json_content = ( + json_content.replace("{json_content}' + ) + + # Insert before if it exists, otherwise before + if "" not in html: + raise ValueError( + "HTML does not contain tag. Cannot inject markdown content." + ) + return html.replace("", f"{script_content}") + +def on_post_page(html: str, page: Page, config: MkDocsConfig) -> str: """Inject Google Tag Manager noscript tag immediately after . Args: - output: The HTML output of the page. + html: The HTML output of the page. page: The page instance. config: The MkDocs configuration object. Returns: modified HTML output with GTM code injected. """ - return _inject_gtm(output) - + html = _inject_markdown_into_html(html, page) + return _inject_gtm(html) # Create HTML files for redirects after site dir has been built def on_post_build(config): diff --git a/docs/docs/agents/models.md b/docs/docs/agents/models.md index 6b8af56a7..46db9c41d 100644 --- a/docs/docs/agents/models.md +++ b/docs/docs/agents/models.md @@ -7,7 +7,7 @@ LangGraph provides built-in support for [LLMs (language models)](https://python. Use [`init_chat_model`](https://python.langchain.com/docs/how_to/chat_models_universal_init/) to initialize models: -{!snippets/chat_model_tabs.md!} +{% include-markdown "../../snippets/chat_model_tabs.md" %} ### Instantiate a model directly diff --git a/docs/docs/how-tos/graph-api.md b/docs/docs/how-tos/graph-api.md index 4a71cc77d..6fa8bd241 100644 --- a/docs/docs/how-tos/graph-api.md +++ b/docs/docs/how-tos/graph-api.md @@ -1507,7 +1507,7 @@ Because many LangChain objects implement the [Runnable Protocol](https://python. See example below. To demonstrate async invocations of underlying LLMs, we will include a chat model: -{!snippets/chat_model_tabs.md!} +{% include-markdown "../../snippets/chat_model_tabs.md" %} ```python from langchain.chat_models import init_chat_model diff --git a/docs/docs/how-tos/memory/semantic-search.ipynb b/docs/docs/how-tos/memory/semantic-search.ipynb index 890363ef4..c952ac6ea 100644 --- a/docs/docs/how-tos/memory/semantic-search.ipynb +++ b/docs/docs/how-tos/memory/semantic-search.ipynb @@ -125,7 +125,7 @@ "memories = store.search((\"user_123\", \"memories\"), query=\"I like food?\", limit=5)\n", "\n", "for memory in memories:\n", - " print(f'Memory: {memory.value[\"text\"]} (similarity: {memory.score})')" + " print(f\"Memory: {memory.value['text']} (similarity: {memory.score})\")" ] }, { diff --git a/docs/docs/index.md b/docs/docs/index.md index 60ec1b509..db036367a 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -28,4 +28,4 @@ title: LangGraph } -{!../README.md!} \ No newline at end of file +{% include-markdown "../../README.md" %} \ No newline at end of file diff --git a/docs/docs/snippets/chat_model_tabs.md b/docs/docs/snippets/chat_model_tabs.md new file mode 100644 index 000000000..e984e1821 --- /dev/null +++ b/docs/docs/snippets/chat_model_tabs.md @@ -0,0 +1,87 @@ +=== "OpenAI" + + ```shell + pip install -U "langchain[openai]" + ``` + ```python + import os + from langchain.chat_models import init_chat_model + + os.environ["OPENAI_API_KEY"] = "sk-..." + + llm = init_chat_model("openai:gpt-4.1") + ``` + + 👉 Read the [OpenAI integration docs](https://python.langchain.com/docs/integrations/chat/openai/) + +=== "Anthropic" + + ```shell + pip install -U "langchain[anthropic]" + ``` + ```python + import os + from langchain.chat_models import init_chat_model + + os.environ["ANTHROPIC_API_KEY"] = "sk-..." + + llm = init_chat_model("anthropic:claude-3-5-sonnet-latest") + ``` + + 👉 Read the [Anthropic integration docs](https://python.langchain.com/docs/integrations/chat/anthropic/) + +=== "Azure" + + ```shell + pip install -U "langchain[openai]" + ``` + ```python + import os + from langchain.chat_models import init_chat_model + + os.environ["AZURE_OPENAI_API_KEY"] = "..." + os.environ["AZURE_OPENAI_ENDPOINT"] = "..." + os.environ["OPENAI_API_VERSION"] = "2025-03-01-preview" + + llm = init_chat_model( + "azure_openai:gpt-4.1", + azure_deployment=os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"], + ) + ``` + + 👉 Read the [Azure integration docs](https://python.langchain.com/docs/integrations/chat/azure_chat_openai/) + +=== "Google Gemini" + + ```shell + pip install -U "langchain[google-genai]" + ``` + ```python + import os + from langchain.chat_models import init_chat_model + + os.environ["GOOGLE_API_KEY"] = "..." + + llm = init_chat_model("google_genai:gemini-2.0-flash") + ``` + + 👉 Read the [Google GenAI integration docs](https://python.langchain.com/docs/integrations/chat/google_generative_ai/) + +=== "AWS Bedrock" + + ```shell + pip install -U "langchain[aws]" + ``` + ```python + from langchain.chat_models import init_chat_model + + # Follow the steps here to configure your credentials: + # https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html + + llm = init_chat_model( + "anthropic.claude-3-5-sonnet-20240620-v1:0", + model_provider="bedrock_converse", + ) + ``` + + 👉 Read the [AWS Bedrock integration docs](https://python.langchain.com/docs/integrations/chat/bedrock/) diff --git a/docs/docs/tutorials/get-started/1-build-basic-chatbot.md b/docs/docs/tutorials/get-started/1-build-basic-chatbot.md index 3304822c6..b32e42861 100644 --- a/docs/docs/tutorials/get-started/1-build-basic-chatbot.md +++ b/docs/docs/tutorials/get-started/1-build-basic-chatbot.md @@ -63,7 +63,7 @@ Next, add a "`chatbot`" node. **Nodes** represent units of work and are typicall Let's first select a chat model: -{!snippets/chat_model_tabs.md!} +{% include-markdown "../../../snippets/chat_model_tabs.md" %}