diff --git a/docs/_scripts/handle_auto_links.py b/docs/_scripts/handle_auto_links.py new file mode 100644 index 000000000..c5c9e4b5d --- /dev/null +++ b/docs/_scripts/handle_auto_links.py @@ -0,0 +1,156 @@ +"""Logic to identify and transform cross-reference links in markdown files. + +This module allows supporting custom markdown syntax for "autolinks". These are links +that will be transformed based on the current scope context, such as "global", "python", +or "js" into an appropriate markdown link format. + +For example, + +```markdown +@[StateGraph] +``` + +May be transformed into: + +```markdown +[StateGraph](some_path/api-reference/state-graph.md) +``` + +The transformation value depends on the scope in which the link is used. +""" + +import logging +import re + +from _scripts.link_map import SCOPE_LINK_MAPS + +logger = logging.getLogger(__name__) + + +def _transform_link( + link_name: str, scope: str, file_path: str, line_number: int +) -> str | None: + """Transform a cross-reference link based on the current scope. + + Args: + link_name: The name of the link to transform (e.g., "StateGraph"). + scope: The current scope context ("global", "python", "js", etc.). + file_path: The file path for error reporting. + line_number: The line number for error reporting. + + Returns: + A formatted markdown link if the link is found in the scope mapping, + None otherwise. + + Example: + >>> _transform_link("StateGraph", "python", "file.md", 5) + "[StateGraph](https://langchain-ai.github.io/langgraph/reference/graphs/#langgraph.graph.StateGraph)" + + >>> _transform_link("unknown-link", "python", "file.md", 5) + None + """ + if scope == "global": + # Special scope that is composed of both Python and JS links + # For now, we will substitute in the python scope! + # But we need to add support for handling both scopes. + scope = "python" + logger.error( + "Encountered unhandled 'global' scope. Defaulting to 'python'." + "In file: %s, line %d, link_name: %s", + file_path, + line_number, + link_name, + ) + link_map = SCOPE_LINK_MAPS.get(scope, {}) + url = link_map.get(link_name) + + if url: + return f"[{link_name}]({url})" + else: + # Log error with file location information + logger.info( + # Using %s + "Link '%s' not found in scope '%s'. " + "In file: %s, line %d. Available links in scope: %s", + link_name, + scope, + file_path, + line_number, + list(link_map.keys() if link_map else []), + ) + return None + + +CONDITIONAL_FENCE_PATTERN = re.compile( + r""" + ^ # Start of line + (?P[ \t]*) # Optional indentation (spaces or tabs) + ::: # Literal fence marker + (?P\w+)? # Optional language identifier (named group: language) + \s* # Optional trailing whitespace + $ # End of line + """, + re.VERBOSE, +) +CROSS_REFERENCE_PATTERN = re.compile( + r""" + @ # Literal @ symbol + \[ # Opening bracket + (?P[^\]]+) # Link name - one or more non-bracket characters + \] # Closing bracket + """, + re.VERBOSE, +) + + +def _replace_autolinks(markdown: str, file_path: str) -> str: + """Preprocess markdown lines to handle @[links] with conditional fence scopes. + + This function processes markdown content to transform @[link_name] references + based on the current conditional fence scope. Conditional fences use the + syntax :::language to define scope boundaries. + + Args: + markdown: The markdown content to process. + file_path: The file path for error reporting. + + Returns: + Processed markdown content with @[references] transformed to proper + markdown links or left unchanged if not found. + + Example: + Input: + "@[StateGraph]\\n:::python\\n@[Command]\\n:::\\n" + Output: + "[StateGraph](url)\\n:::python\\n[Command](url)\\n:::\\n" + """ + # Track the current scope context + current_scope = "global" + lines = markdown.splitlines(keepends=True) + processed_lines = [] + + for line_number, line in enumerate(lines, 1): + line_stripped = line.strip() + + # Check if this line defines a new conditional fence scope + fence_match = CONDITIONAL_FENCE_PATTERN.match(line_stripped) + if fence_match: + language = fence_match.group("language") + # Set scope to the specified language, or reset to global if no language + current_scope = language.lower() if language else "global" + processed_lines.append(line) + continue + + # Transform all @[link_name] references in this line based on current scope + def replace_cross_reference(match: re.Match[str]) -> str: + """Replace a single @[link_name] with the scoped equivalent.""" + link_name = match.group("link_name") + transformed = _transform_link( + link_name, current_scope, file_path, line_number + ) + return transformed if transformed is not None else match.group(0) + + transformed_line = CROSS_REFERENCE_PATTERN.sub(replace_cross_reference, line) + processed_lines.append(transformed_line) + + return "".join(processed_lines) diff --git a/docs/_scripts/link_map.py b/docs/_scripts/link_map.py index de4f2526f..4d474b7a8 100644 --- a/docs/_scripts/link_map.py +++ b/docs/_scripts/link_map.py @@ -1,5 +1,28 @@ -JS_LINK_MAP = { - "langgraph.types.interrupt": "https://langchain-ai.github.io/langgraphjs/reference/functions/langgraph.interrupt-2.html", - "create_react_agent": "https://langchain-ai.github.io/langgraphjs/reference/functions/langgraph_prebuilt.createReactAgent.html", - "langgraph.types.Command": "https://langchain-ai.github.io/langgraphjs/reference/classes/langgraph.Command.html", +"""Link mapping for cross-reference resolution across different scopes. + +This module provides link mappings for different language/framework scopes +to resolve @[link_name] references to actual URLs. +""" +# Python-specific link mappings +PYTHON_LINK_MAP = { + "StateGraph": "https://langchain-ai.github.io/langgraph/reference/graphs/#langgraph.graph.StateGraph", + "interrupt": "https://langchain-ai.github.io/langgraph/reference/graphs/#langgraph.graph.interrupt", + "create_react_agent": "https://langchain-ai.github.io/langgraph/reference/prebuilt/#langgraph.prebuilt.create_react_agent", + "Command": "https://langchain-ai.github.io/langgraph/reference/types/#langgraph.types.Command", +} + +# JavaScript-specific link mappings +JS_LINK_MAP = { + "StateGraph": "https://langchain-ai.github.io/langgraphjs/reference/classes/langgraph.StateGraph.html", + "interrupt": "https://langchain-ai.github.io/langgraphjs/reference/functions/langgraph.interrupt-2.html", + "create_react_agent": "https://langchain-ai.github.io/langgraphjs/reference/functions/langgraph_prebuilt.createReactAgent.html", + "Command": "https://langchain-ai.github.io/langgraphjs/reference/classes/langgraph.Command.html", +} + + +# Global scope is assembled from the Python and JS mappings +# Combined mapping by scope +SCOPE_LINK_MAPS = { + "python": PYTHON_LINK_MAP, + "js": JS_LINK_MAP, } diff --git a/docs/_scripts/notebook_hooks.py b/docs/_scripts/notebook_hooks.py index 81882d397..6400a0da3 100644 --- a/docs/_scripts/notebook_hooks.py +++ b/docs/_scripts/notebook_hooks.py @@ -16,7 +16,7 @@ from mkdocs.structure.files import Files, File from mkdocs.structure.pages import Page from _scripts.generate_api_reference_links import update_markdown_with_imports -from _scripts.link_map import JS_LINK_MAP +from _scripts.handle_auto_links import _replace_autolinks from _scripts.notebook_convert import convert_notebook logger = logging.getLogger(__name__) @@ -176,31 +176,7 @@ def _add_path_to_code_blocks(markdown: str, page: Page) -> str: return code_block_pattern.sub(replace_code_block_header, markdown) -def _resolve_cross_references(md_text: str, link_map: dict[str, str]) -> str: - """Replace [title][identifier] with [title](url) using language-specific link_map. - - Args: - md_text: The markdown text to process. - link_map: mapping of identifier to URL. - - Returns: - The processed markdown text with cross-references resolved. - """ - # Pattern to match [title][identifier] - pattern = re.compile(r"\[([^\]]+)\]\[([^\]]+)\]") - - def replace_reference(match: re.Match) -> str: - """Replace the matched reference with the corresponding URL.""" - title, identifier = match.group(1), match.group(2) - url = link_map.get(identifier) - - if url: - return f"[{title}]({url})" - else: - # Leave it unchanged if not found - return match.group(0) - - return pattern.sub(replace_reference, md_text) +# Compiled regex patterns for better performance and readability def _apply_conditional_rendering(md_text: str, target_language: str) -> str: @@ -295,7 +271,7 @@ def _highlight_code_blocks(markdown: str) -> str: opening_fence += f" {attributes}" if highlighted_lines: - opening_fence += f" hl_lines=\"{' '.join(highlighted_lines)}\"" + opening_fence += f' hl_lines="{" ".join(highlighted_lines)}"' return ( # The indent and opening fence @@ -325,6 +301,9 @@ def _on_page_markdown_with_config( # logger.info("Processing Jupyter notebook: %s", page.file.src_path) markdown = convert_notebook(page.file.abs_src_path) + # Apply cross-reference preprocessing to all markdown content + markdown = _replace_autolinks(markdown, page.file.src_path) + # Append API reference links to code blocks if add_api_references: markdown = update_markdown_with_imports(markdown, page.file.abs_src_path) @@ -334,16 +313,6 @@ def _on_page_markdown_with_config( # Apply conditional rendering for code blocks target_language = kwargs.get("target_language", "python") markdown = _apply_conditional_rendering(markdown, target_language) - if target_language == "js": - markdown = _resolve_cross_references(markdown, JS_LINK_MAP) - elif target_language == "python": - # Via a dedicated plugin - pass - else: - raise ValueError( - f"Unsupported target language: {target_language}. " - "Supported languages are 'python' and 'js'." - ) # Add file path as an attribute to code blocks that are executable. # This file path is used to associate fixtures with the executable code @@ -358,13 +327,11 @@ def _on_page_markdown_with_config( def on_page_markdown(markdown: str, page: Page, **kwargs: Dict[str, Any]): - finalized_markdown = ( - _on_page_markdown_with_config( - markdown, - page, - add_api_references=True, - **kwargs, - ) + finalized_markdown = _on_page_markdown_with_config( + markdown, + page, + add_api_references=True, + **kwargs, ) page.meta["original_markdown"] = finalized_markdown return finalized_markdown @@ -437,6 +404,7 @@ height="0" width="0" style="display:none;visibility:hidden"> else: return html # fallback if no found + def _inject_markdown_into_html(html: str, page: Page) -> str: """Inject the original markdown content into the HTML page as JSON.""" original_markdown = page.meta.get("original_markdown", "") @@ -469,6 +437,7 @@ def _inject_markdown_into_html(html: str, page: Page) -> str: ) return html.replace("", f"{script_content}") + def on_post_page(html: str, page: Page, config: MkDocsConfig) -> str: """Inject Google Tag Manager noscript tag immediately after . @@ -483,6 +452,7 @@ def on_post_page(html: str, page: Page, config: MkDocsConfig) -> str: html = _inject_markdown_into_html(html, page) return _inject_gtm(html) + # Create HTML files for redirects after site dir has been built def on_post_build(config): use_directory_urls = config.get("use_directory_urls") diff --git a/docs/tests/unit_tests/test_auto_links.py b/docs/tests/unit_tests/test_auto_links.py new file mode 100644 index 000000000..3a7b34aa2 --- /dev/null +++ b/docs/tests/unit_tests/test_auto_links.py @@ -0,0 +1,146 @@ +"""Unit tests for cross-reference preprocessing functionality.""" + +from unittest.mock import patch + +import pytest + +from _scripts.handle_auto_links import _transform_link, _replace_autolinks + + +@pytest.fixture +def mock_link_maps(): + """Fixture providing mock link maps for testing.""" + mock_scope_maps = { + "python": {"py-link": "https://example.com/python"}, + "js": {"js-link": "https://example.com/js"}, + } + + with patch("_scripts.handle_auto_links.SCOPE_LINK_MAPS", mock_scope_maps): + yield mock_scope_maps + + +def test_transform_link_basic(mock_link_maps) -> None: + """Test basic link transformation.""" + # Test with a known link + result = _transform_link("py-link", "python", "test.md", 1) + assert result == "[py-link](https://example.com/python)" + + # Test with an unknown link (returns None) + result = _transform_link("unknown-link", "global", "test.md", 1) + assert result is None + + +def test_no_cross_refs(mock_link_maps) -> None: + """Test markdown with no @[references].""" + lines = ["# Title\n", "Regular text.\n"] + markdown = "".join(lines) + result = _replace_autolinks(markdown, "test.md") + expected = "".join(["# Title\n", "Regular text.\n"]) + assert result == expected + + +def test_global_cross_refs(mock_link_maps) -> None: + """Test @[references] in global scope (no conditional blocks).""" + lines = ["@[global-link]\n", "Text with @[unknown-link].\n"] + markdown = "".join(lines) + result = _replace_autolinks(markdown, "test.md") + expected = "".join(["@[global-link]\n", "Text with @[unknown-link].\n"]) + assert result == expected + + +def test_python_conditional_block(mock_link_maps) -> None: + """Test @[references] inside Python conditional block.""" + lines = [":::python\n", "@[py-link]\n", ":::\n"] + markdown = "".join(lines) + result = _replace_autolinks(markdown, "test.md") + expected = "".join( + [":::python\n", "[py-link](https://example.com/python)\n", ":::\n"] + ) + assert result == expected + + +def test_js_conditional_block(mock_link_maps) -> None: + """Test @[references] inside JavaScript conditional block.""" + lines = [":::js\n", "@[js-link]\n", ":::\n"] + markdown = "".join(lines) + result = _replace_autolinks(markdown, "test.md") + expected = "".join([":::js\n", "[js-link](https://example.com/js)\n", ":::\n"]) + assert result == expected + + +def test_all_scopes(mock_link_maps) -> None: + """Test @[references] in global, Python, and JavaScript scopes.""" + lines = [ + "@[global-link]\n", + ":::python\n", + "@[py-link]\n", + ":::\n", + "@[global-link]\n", + ":::js\n", + "@[js-link]\n", + ":::\n", + "@[global-link]\n", + ] + markdown = "".join(lines) + result = _replace_autolinks(markdown, "test.md") + expected = "".join( + [ + "@[global-link]\n", + ":::python\n", + "[py-link](https://example.com/python)\n", + ":::\n", + "@[global-link]\n", + ":::js\n", + "[js-link](https://example.com/js)\n", + ":::\n", + "@[global-link]\n", + ] + ) + assert result == expected + + +def test_fence_resets_to_global(mock_link_maps) -> None: + """Test that closing fence resets scope to global.""" + lines = [":::python\n", "@[py-link]\n", ":::\n", "@[global-link]\n"] + markdown = "".join(lines) + result = _replace_autolinks(markdown, "test.md") + expected = "".join( + [ + ":::python\n", + "[py-link](https://example.com/python)\n", + ":::\n", + "@[global-link]\n", + ] + ) + assert result == expected + + +def test_indented_conditional_fences(mock_link_maps) -> None: + """Test @[references] inside indented conditional fences (e.g., in tabs or admonitions).""" + lines = [ + "@[global-link]\n", + " :::python\n", + " @[py-link]\n", + " :::\n", + "@[global-link]\n", + "\t\t:::js\n", + "\t\t@[js-link]\n", + "\t\t:::\n", + "@[global-link]\n", + ] + markdown = "".join(lines) + result = _replace_autolinks(markdown, "test.md") + expected = "".join( + [ + "@[global-link]\n", + " :::python\n", + " [py-link](https://example.com/python)\n", + " :::\n", + "@[global-link]\n", + "\t\t:::js\n", + "\t\t[js-link](https://example.com/js)\n", + "\t\t:::\n", + "@[global-link]\n", + ] + ) + assert result == expected