From f0f329d9e12803a3f5bc21c48e362eee63d248fe Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Fri, 20 Jun 2025 16:20:16 -0400 Subject: [PATCH] docs: Add conditional js/python rendering (#5128) # Overview Adding conditional rendering logic to co-locate js and python documentation. * `:::` conditional syntax can be used to switch between python only or js only content. * Contains simple unit tests for `:::` * PR adds set up for a way to implement a context switch between languages, but it will not be enabled until JS content is merged in. * Contains a script that can add javascript documentation Implementation of: https://github.com/langchain-ai/langgraph/pull/5118 ## Example Example of conditional rendering / compilation. ```markdown ### Config (static context) Config is for immutable data like user metadata or API keys. Use when you have values that don't change mid-run. Specify configuration using a key called **"configurable"** which is reserved for this purpose: :::python This content will only be rendered for the python site. ::: :::js this content will only be rendered for the js / ts site. ::: ``` --- docs/_scripts/add_translation.py | 241 +++++++++--------- docs/_scripts/generate_llms_text.py | 16 +- docs/_scripts/link_map.py | 5 + docs/_scripts/notebook_hooks.py | 71 ++++++ docs/overrides/language-switcher.css | 41 +++ docs/overrides/language-switcher.js | 38 +++ docs/overrides/partials/language-toggle.html | 6 + .../unit_tests/test_conditional_rendering.py | 22 ++ 8 files changed, 312 insertions(+), 128 deletions(-) create mode 100644 docs/_scripts/link_map.py create mode 100644 docs/overrides/language-switcher.css create mode 100644 docs/overrides/language-switcher.js create mode 100644 docs/overrides/partials/language-toggle.html create mode 100644 docs/tests/unit_tests/test_conditional_rendering.py diff --git a/docs/_scripts/add_translation.py b/docs/_scripts/add_translation.py index 34d9da416..a02812bc9 100644 --- a/docs/_scripts/add_translation.py +++ b/docs/_scripts/add_translation.py @@ -1,157 +1,154 @@ -"""Add typescript translation to a given markdown file.""" +"""Translate Python markdown to TypeScript and/or consolidate Python-JS markdown into a single document.""" import argparse -import re import requests from langchain_anthropic import ChatAnthropic +# Load reference TypeScript snippets URL = "https://gist.githubusercontent.com/eyurtsev/e7486731415463a9bc5b4682358859c8/raw/b5a5fda9c7e3387cfcb781f25082814d43675d50/gistfile1.txt" response = requests.get(URL) response.raise_for_status() reference_snippets = response.text -model = ChatAnthropic(model="claude-3-5-sonnet-latest") +# Initialize model +model = ChatAnthropic(model="claude-sonnet-4-0", max_tokens=64_000) + +TRANSLATION_PROMPT = ( + "You are a helpful assistant that translates Python-based technical " + "documentation written in Markdown to equivalent TypeScript-based documentation. " + "The input is a Markdown file written in mkdocs format. It contains " + "Python code snippets embedded in prose. " + "Your task is to rewrite the content by translating the Python code to " + "idiomatic TypeScript, using the provided TypeScript reference snippets " + "to ensure accurate and consistent usage (e.g., correct imports, function " + "names, and patterns). " + "Remove the original Python code and replace it with the corresponding " + "TypeScript version. " + "Do not alter the surrounding prose unless a change is necessary to " + "reflect differences between Python and TypeScript. " + "Preserve the structure and formatting of the original Markdown document. " + "Do not make stylistic or structural changes unless they directly support " + "the translation. " + "Use the reference TypeScript snippets as guidance whenever possible to " + "maintain alignment with existing conventions.\n\n" + f"Here are the reference TypeScript snippets:\n\n{reference_snippets}\n\n" +) + +CONSOLIDATION_PROMPT = ( + "You are a helpful assistant that consolidates parallel Python and JavaScript (TypeScript) technical documentation " + "written in Markdown into a single unified Markdown document. " + "The input consists of two documents: the first is for Python users, and the second is for JavaScript/TypeScript users. " + "Your task is to merge these into one Markdown file using language-specific fenced blocks to separate the content where needed. " + "Use the following syntax to distinguish content for each language:\n\n" + ":::python\n" + "# Python-specific content\n" + ":::\n\n" + ":::js\n" + "# JavaScript/TypeScript-specific content\n" + ":::\n\n" + "Follow these consolidation rules:\n" + "- When content (prose or code) is the same or nearly identical in both versions, include it only once—outside of any fenced block.\n" + "- When content differs between the Python and JS versions, wrap each version in its corresponding fenced block.\n" + "- Prefer **paragraph-level separation** of language-specific content. Do not combine Python and JS snippets or terminology in the same sentence or paragraph using conditional phrases.\n" + " For example, avoid inline constructs like:\n" + " `The :::python add_messages ::: :::js reducer ::: function...`\n" + " Instead, write two distinct paragraphs:\n\n" + " :::python\n" + " The `add_messages` function in our `State` will append the LLM's response messages to whatever messages are already in the state.\n" + " ::: \n\n" + " :::js\n" + " The `reducer` function in our `StateAnnotation` will append the LLM's response messages to whatever messages are already in the state.\n" + " :::\n\n" + "- Preserve the overall structure, ordering, and formatting of the original Markdown documents.\n" + "- Do not rephrase or unify content unless it is logically and semantically identical.\n" + "- Use the fenced blocks for both prose and code as needed, and ensure output is clean, readable Markdown suitable for tools that parse these directives.\n" + "Your goal is to produce a cleanly merged documentation file that serves both Python and JavaScript users without redundancy, while maximizing clarity and separation of language-specific details." +) -def _get_tqdm(): - try: - from tqdm import tqdm - except ImportError: - # If not available return a simple identity function - def tqdm(iterable, *args, **kwargs): - return iterable - - return tqdm - - -_tqdm = _get_tqdm() - -opening_pattern = re.compile(r"^\s*```python(?:\s+.*)?\s*$") -closing_pattern = re.compile(r"^\s*```\s*$") - - -def extract_python_snippets(markdown: str) -> list[str]: - """ - Extract all python code blocks (including their fence lines) from the markdown content. - A python block is defined as any block that starts with a line containing an opening fence - with '```python' (optionally with extra parameters) and ends with a closing fence '```'. - """ - snippets = [] - inside_block = False - current_snippet = [] - - for line in markdown.splitlines(keepends=True): - if not inside_block: - if opening_pattern.match(line): - inside_block = True - current_snippet = [line] - else: - current_snippet.append(line) - if closing_pattern.match(line): - inside_block = False - snippets.append("".join(current_snippet)) - current_snippet = [] - return snippets - - -def translate_snippet(python_snippet: str) -> str: - """Translate a python code block into a TypeScript code block using Langchain. - The response is expected to be a properly fenced TypeScript code block (i.e. - starting with ```typescript and ending with ```). - """ - ai_message = model.invoke( +def translate_python_to_ts(markdown_content: str) -> str: + response = model.invoke( [ { "role": "system", - "content": ( - f"You have access to the following up-to-date example TypeScript code " - f"snippets that show examples of building with langgraph " - f"and langchain:\n\n{reference_snippets}\n\n" - "Use this context to translate the following Python code to equivalent " - "TypeScript. Ensure that your output is a valid fenced TypeScript " - "code block (i.e. starts with ```typescript and ends with ```)." - ), - }, - { - "role": "user", - "content": f"Translate this Python snippet to TypeScript:\n\n{python_snippet}", + "content": TRANSLATION_PROMPT, + "cache_control": {"type": "ephemeral"}, }, + {"role": "user", "content": markdown_content}, ] ) - - # Use a regular expression to search for a TypeScript code block in the response. - pattern = r"```typescript\s*(.*?)\s*```" - match = re.search(pattern, ai_message.content, re.DOTALL) - if match: - # Reconstruct the code block with proper fences. - typescript_code = match.group(1).strip() - return f"```typescript\n{typescript_code}\n```" - else: - raise ValueError("No TypeScript code block found in the model's response.") + return response.content -def insert_translations_into_markdown( - markdown: str, typescript_snippets: list[str] -) -> str: - """Walks through the original markdown content and, after each - Python snippet block, inserts the corresponding translated TypeScript snippet. - It assumes that the ordering of the Python snippets - (from extract_python_snippets) matches the order they appear in the markdown. - """ - output_lines = [] - lines = markdown.splitlines(keepends=True) - inside_block = False - snippet_index = 0 - - for line in lines: - output_lines.append(line) - if not inside_block and opening_pattern.match(line): - # We've encountered the start of a python code block. - inside_block = True - elif inside_block: - if closing_pattern.match(line): - # End of a python snippet block. - inside_block = False - if snippet_index < len(typescript_snippets): - # Insert an extra newline for clarity, then the translated TypeScript snippet. - output_lines.append("\n") - output_lines.append(typescript_snippets[snippet_index]) - output_lines.append("\n") - snippet_index += 1 - return "".join(output_lines) +def consolidate_python_and_ts(combined_content: str) -> str: + response = model.invoke( + [ + { + "role": "system", + "content": CONSOLIDATION_PROMPT, + "cache_control": {"type": "ephemeral"}, + }, + {"role": "user", "content": combined_content}, + ] + ) + return response.content -def main(file_path: str) -> None: - # Read the markdown file. - with open(file_path, "r") as f: +def main(file_path: str, translate_only: bool, consolidate_only: bool) -> None: + with open(file_path, "r", encoding="utf-8") as f: markdown_content = f.read() - # 1. Extract all Python snippets. - python_snippets = extract_python_snippets(markdown_content)[:1] + if translate_only: + translated = translate_python_to_ts(markdown_content) + output_path = file_path.replace(".md", ".translated.md") + with open(output_path, "w", encoding="utf-8") as f: + f.write(translated) + print(f"Translated JS/TS version written to: {output_path}") - # 2. Translate each Python snippet to TypeScript. - typescript_snippets = [] - # Replace with .batch() for faster translation - for python_snippet in _tqdm(python_snippets): - ts_snippet = translate_snippet(python_snippet) - typescript_snippets.append(ts_snippet) + elif consolidate_only: + consolidated = consolidate_python_and_ts(markdown_content) + with open(file_path, "w", encoding="utf-8") as f: + f.write(consolidated) + print(f"Consolidated content written to: {file_path}") - # 3. Insert the TypeScript translations after their respective Python snippets. - updated_markdown = insert_translations_into_markdown( - markdown_content, typescript_snippets - ) - - # Overwrite the original markdown file with the updated content. - with open(file_path, "w") as f: - f.write(updated_markdown) + else: + # Default behavior: translate first, then consolidate both + translated = translate_python_to_ts(markdown_content) + combined = f"{markdown_content.strip()}\n\n\n{translated.strip()}" + consolidated = consolidate_python_and_ts(combined) + with open(file_path, "w", encoding="utf-8") as f: + f.write(consolidated) + print(f"Translated and consolidated content written to: {file_path}") if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Translate Python snippets in a markdown file to TypeScript and insert them after each Python snippet." + description=( + "Translate Python markdown to TypeScript and/or consolidate " + "Python-JS markdown into one file." + ) ) parser.add_argument("file_path", type=str, help="Path to the markdown file.") + parser.add_argument( + "--translate-only", + action="store_true", + help="Only generate the JS translation.", + ) + parser.add_argument( + "--consolidate-only", + action="store_true", + help="Only consolidate pre-paired Python and JS content.", + ) args = parser.parse_args() - main(args.file_path) + if args.translate_only and args.consolidate_only: + raise ValueError( + "Cannot use both --translate-only and --consolidate-only at the same time." + ) + + main( + args.file_path, + translate_only=args.translate_only, + consolidate_only=args.consolidate_only, + ) diff --git a/docs/_scripts/generate_llms_text.py b/docs/_scripts/generate_llms_text.py index d71f28589..ff981834e 100644 --- a/docs/_scripts/generate_llms_text.py +++ b/docs/_scripts/generate_llms_text.py @@ -3,19 +3,21 @@ import asyncio import glob import os -from typing import TypedDict, List, Optional -import pydantic import re -from pydantic import BaseModel, Field -from langchain_core.rate_limiters import InMemoryRateLimiter +from typing import TypedDict, List, Optional import yaml from langchain.chat_models import init_chat_model +from langchain_core.rate_limiters import InMemoryRateLimiter from mkdocs.structure.files import File from mkdocs.structure.pages import Page +from pydantic import BaseModel, Field from yaml import SafeLoader -from _scripts.notebook_hooks import _on_page_markdown_with_config +from _scripts.notebook_hooks import ( + _on_page_markdown_with_config, + _apply_conditional_rendering, +) HERE = os.path.dirname(os.path.abspath(__file__)) # Get source directory (parent of HERE / docs) @@ -211,7 +213,9 @@ async def process_nav_items(nav_items: list[NavItem]) -> list[NavItem]: # Remove any items that start with http:// or https:// looking only for # local file at this stages. nav_items = [ - item for item in nav_items if not item["url"].startswith(("http://", "https://")) + item + for item in nav_items + if not item["url"].startswith(("http://", "https://")) ] # Process items in parallel tasks = [process_single_item(item) for item in nav_items] diff --git a/docs/_scripts/link_map.py b/docs/_scripts/link_map.py new file mode 100644 index 000000000..de4f2526f --- /dev/null +++ b/docs/_scripts/link_map.py @@ -0,0 +1,5 @@ +JS_LINK_MAP = { + "langgraph.types.interrupt": "https://langchain-ai.github.io/langgraphjs/reference/functions/langgraph.interrupt-2.html", + "create_react_agent": "https://langchain-ai.github.io/langgraphjs/reference/functions/langgraph_prebuilt.createReactAgent.html", + "langgraph.types.Command": "https://langchain-ai.github.io/langgraphjs/reference/classes/langgraph.Command.html", +} diff --git a/docs/_scripts/notebook_hooks.py b/docs/_scripts/notebook_hooks.py index 30545408e..988fc21d9 100644 --- a/docs/_scripts/notebook_hooks.py +++ b/docs/_scripts/notebook_hooks.py @@ -16,6 +16,7 @@ from mkdocs.structure.pages import Page from _scripts.generate_api_reference_links import update_markdown_with_imports from _scripts.notebook_convert import convert_notebook +from _scripts.link_map import JS_LINK_MAP logger = logging.getLogger(__name__) logging.basicConfig() @@ -158,6 +159,62 @@ def _add_path_to_code_blocks(markdown: str, page: Page) -> str: return code_block_pattern.sub(replace_code_block_header, markdown) +def _resolve_cross_references(md_text: str, link_map: dict[str, str]) -> str: + """Replace [title][identifier] with [title](url) using language-specific link_map. + + Args: + md_text: The markdown text to process. + link_map: mapping of identifier to URL. + + Returns: + The processed markdown text with cross-references resolved. + """ + # Pattern to match [title][identifier] + pattern = re.compile(r"\[([^\]]+)\]\[([^\]]+)\]") + + def replace_reference(match: re.Match) -> str: + """Replace the matched reference with the corresponding URL.""" + title, identifier = match.group(1), match.group(2) + url = link_map.get(identifier) + + if url: + return f"[{title}]({url})" + else: + # Leave it unchanged if not found + return match.group(0) + + return pattern.sub(replace_reference, md_text) + + +def _apply_conditional_rendering(md_text: str, target_language: str) -> str: + if target_language not in {"python", "js"}: + raise ValueError("target_language must be 'python' or 'js'") + + pattern = re.compile( + r"(?P[ \t]*):::(?P\w+)\s*\n" + r"(?P((?:.*\n)*?))" # Capture the content inside the block + r"(?P=indent):::" # Match closing with the same indentation + ) + + def replace_conditional_blocks(match: re.Match) -> str: + """Keep active conditionals.""" + language = match.group("language") + content = match.group("content") + + if language not in {"python", "js"}: + # If the language is not supported, return the original block + return match.group(0) + + if language == target_language: + return content + + # If the language does not match, return an empty string + return "" + + processed = pattern.sub(replace_conditional_blocks, md_text) + return processed + + def _highlight_code_blocks(markdown: str) -> str: """Find code blocks with highlight comments and add hl_lines attribute. @@ -257,6 +314,20 @@ def _on_page_markdown_with_config( # Apply highlight comments to code blocks markdown = _highlight_code_blocks(markdown) + # Apply conditional rendering for code blocks + target_language = kwargs.get("target_language", "python") + markdown = _apply_conditional_rendering(markdown, target_language) + if target_language == "js": + markdown = _resolve_cross_references(markdown, JS_LINK_MAP) + elif target_language == "python": + # Via a dedicated plugin + pass + else: + raise ValueError( + f"Unsupported target language: {target_language}. " + "Supported languages are 'python' and 'js'." + ) + # Add file path as an attribute to code blocks that are executable. # This file path is used to associate fixtures with the executable code # which can be used in CI to test the docs without making network requests. diff --git a/docs/overrides/language-switcher.css b/docs/overrides/language-switcher.css new file mode 100644 index 000000000..63453ffd9 --- /dev/null +++ b/docs/overrides/language-switcher.css @@ -0,0 +1,41 @@ +.lang-python, +.lang-javascript { + display: none; +} + +.language-switcher-global { + display: flex; + align-items: center; + padding-left: 0.5rem; + margin-right: 0.5rem; +} + +/* Style the select to match the header */ +.language-switcher-global select { + appearance: none; + font: inherit; + border: none; + padding: 0.25rem 0.6rem; + cursor: pointer; + outline: none; + font-weight: bolder; +} + +/* Hover/focus effect */ +.language-switcher-global select:hover, +.language-switcher-global select:focus { + text-decoration: underline; +} + +/* Theme-specific overrides */ +html[data-md-color-scheme="default"] .language-switcher-global select, +html[data-md-color-scheme="default"] .language-switcher-global option { + color: #333; + background-color: transparent; +} + +html[data-md-color-scheme="slate"] .language-switcher-global select, +html[data-md-color-scheme="slate"] .language-switcher-global option { + color: #eee; + background-color: transparent; +} diff --git a/docs/overrides/language-switcher.js b/docs/overrides/language-switcher.js new file mode 100644 index 000000000..51032723b --- /dev/null +++ b/docs/overrides/language-switcher.js @@ -0,0 +1,38 @@ +function applyLanguageSwitching() { + const selector = document.getElementById("global-language-selector"); + + const langBlocks = { + python: document.querySelectorAll(".lang-python"), + javascript: document.querySelectorAll(".lang-javascript"), + }; + + const setLanguage = (lang) => { + for (const [key, blocks] of Object.entries(langBlocks)) { + blocks.forEach((block) => { + block.style.display = key === lang ? "block" : "none"; + }); + } + localStorage.setItem("preferredLang", lang); + }; + + const saved = localStorage.getItem("preferredLang") || "python"; + + if (selector) { + selector.value = saved; + selector.addEventListener("change", (e) => setLanguage(e.target.value)); + } + + setLanguage(saved); +} + +// Run on initial load +document.addEventListener("DOMContentLoaded", applyLanguageSwitching); + +// Re-run after client-side navigation (MkDocs Material) +document.addEventListener("pjax:success", applyLanguageSwitching); + +// Optional: observe DOM changes (e.g., for late-loaded content) +if (window.MutationObserver) { + const observer = new MutationObserver(() => applyLanguageSwitching()); + observer.observe(document.body, { childList: true, subtree: true }); +} diff --git a/docs/overrides/partials/language-toggle.html b/docs/overrides/partials/language-toggle.html new file mode 100644 index 000000000..14eadc529 --- /dev/null +++ b/docs/overrides/partials/language-toggle.html @@ -0,0 +1,6 @@ +
+ +
diff --git a/docs/tests/unit_tests/test_conditional_rendering.py b/docs/tests/unit_tests/test_conditional_rendering.py new file mode 100644 index 000000000..2bfcedd3b --- /dev/null +++ b/docs/tests/unit_tests/test_conditional_rendering.py @@ -0,0 +1,22 @@ +from _scripts.notebook_hooks import _apply_conditional_rendering + + +CONDITIONAL_RENDERING = """ +above +:::js +js-content +::: +between +:::python +python-content +::: +below +""" + + +def test_conditional_rendering() -> None: + """Test logic for conditional rendering of content.""" + output = _apply_conditional_rendering(CONDITIONAL_RENDERING, "js") + assert output.strip() == "above\njs-content\n\nbetween\n\nbelow" + output = _apply_conditional_rendering(CONDITIONAL_RENDERING, "python") + assert output.strip() == "above\n\nbetween\npython-content\n\nbelow"