From 4da35babda464ab6d9ed17d99ae90fec438a047c Mon Sep 17 00:00:00 2001 From: Xin Jin <39755499+EugeneJinXin@users.noreply.github.com> Date: Fri, 11 Jul 2025 12:13:55 -0700 Subject: [PATCH] feat: add copy page button functionality and fix llms-text output (#5419) * feat: add copy page button functionality and fix llms-text output - Add copy page button with CSS and JS implementation - Implement copy page hooks for MkDocs integration - Fix HTML filtering and DOM text reinterpreted as HTML issues - Update llms-text target to generate docs/llm.txt instead of docs/llms-full.txt - Add necessary styling and package.json dependencies * fix missing button in preview * remove the over-processing * disable API reference --- docs/_scripts/copy_page_hooks.py | 162 +++++++++++++++++++++++++++++++ docs/mkdocs.yml | 1 + docs/overrides/copy-page.css | 16 +++ docs/overrides/copy-page.js | 38 ++++++++ docs/overrides/main.html | 135 ++++++++++++++++++++++++++ docs/package.json | 3 +- 6 files changed, 354 insertions(+), 1 deletion(-) create mode 100644 docs/_scripts/copy_page_hooks.py create mode 100644 docs/overrides/copy-page.css create mode 100644 docs/overrides/copy-page.js diff --git a/docs/_scripts/copy_page_hooks.py b/docs/_scripts/copy_page_hooks.py new file mode 100644 index 000000000..2dd42b29c --- /dev/null +++ b/docs/_scripts/copy_page_hooks.py @@ -0,0 +1,162 @@ +""" +Copy page functionality hooks for MkDocs. + +This module provides hooks to inject original markdown content into HTML pages +for the copy page functionality, allowing users to copy clean markdown content +optimized for LLMs. +""" + +import json +import re +from pathlib import Path +from typing import Optional + +from mkdocs.config.defaults import MkDocsConfig +from mkdocs.structure.pages import Page + + +def _process_includes(content: str, docs_dir: Path) -> str: + """Process MkDocs includes like {!../README.md!}.""" + include_pattern = r'\{!([^!]+)!\}' + + def replace_include(match): + include_path = match.group(1) + # Resolve relative path + if include_path.startswith('../'): + # Go up from docs dir + include_file = docs_dir.parent / include_path[3:] + else: + include_file = docs_dir / include_path + + try: + with open(include_file, 'r', encoding='utf-8') as f: + included_content = f.read() + # Remove frontmatter from included content to avoid duplication + included_content = re.sub(r'^---\n.*?\n---\n', '', included_content, flags=re.DOTALL) + return included_content + except: + return f"[Content from {include_path}]" + + return re.sub(include_pattern, replace_include, content) + + +def _clean_markdown(content: str) -> str: + """Minimal cleanup of markdown content - preserve original as much as possible.""" + # Remove frontmatter + content = re.sub(r'^---\n.*?\n---\n', '', content, flags=re.DOTALL) + + # Remove script tags (security) + content = re.sub(r']*>.*?', '', content, flags=re.DOTALL | re.IGNORECASE) + + # Remove style tags (security) + content = re.sub(r']*>.*?', '', content, flags=re.DOTALL | re.IGNORECASE) + + # Remove HTML comments + content = re.sub(r'', '', content, flags=re.DOTALL) + + # Just strip and return - preserve original structure + return content.strip() + + +def inject_markdown_content(html: str, page: Page, config: MkDocsConfig) -> str: + """ + Inject the original markdown content into the HTML for copy page functionality. + + Args: + html: The HTML content to inject into + page: The MkDocs page object + config: The MkDocs configuration + + Returns: + Modified HTML with markdown content injected as JSON + """ + if not hasattr(page, 'file') or not page.file: + return html + + # Get the original markdown file path + docs_dir = Path(config.get('docs_dir', 'docs')) + src_path = page.file.src_path + + # Handle different file types + if src_path.endswith('.ipynb'): + # For notebook files, we might want to use the converted markdown + # For now, just return the HTML as-is + return html + + markdown_file = docs_dir / src_path + + if not markdown_file.exists(): + return html + + try: + # Read the original markdown content + with open(markdown_file, 'r', encoding='utf-8') as f: + markdown_content = f.read() + + # Special handling for index page - use relative path to the actual README.md + if src_path == 'index.md': + # Relative path to the repository README.md file (go up two levels from docs/docs) + readme_path = docs_dir.parent.parent / 'README.md' + + try: + with open(readme_path, 'r', encoding='utf-8') as f: + readme_content = f.read() + # Remove frontmatter if present + processed_markdown = re.sub(r'^---\n.*?\n---\n', '', readme_content, flags=re.DOTALL) + processed_markdown = processed_markdown.strip() + except Exception as e: + # If we can't read the README, fallback to original behavior + processed_markdown = _process_includes(markdown_content, docs_dir) + processed_markdown = re.sub(r'^---\n.*?\n---\n', '', processed_markdown, flags=re.DOTALL) + processed_markdown = processed_markdown.strip() + else: + # Process any includes in the markdown to get the full content + processed_markdown = _process_includes(markdown_content, docs_dir) + # Clean up the processed markdown normally for other pages + processed_markdown = _clean_markdown(processed_markdown) + + # Create the JSON data + markdown_data = { + 'markdown': processed_markdown, + 'title': page.title or 'Page Content', + 'url': page.url or '' + } + + # Properly escape the JSON for HTML + json_content = json.dumps(markdown_data, ensure_ascii=False) + json_content = json_content.replace('{json_content}' + + # Insert before if it exists, otherwise before + if '' in html: + html = html.replace('', f'{script_content}') + elif '' in html: + html = html.replace('', f'{script_content}') + + except Exception as e: + # If anything goes wrong, just return the original HTML + # Could log the error here if needed + pass + + return html + + +def on_post_page(output: str, page: Page, config: MkDocsConfig) -> str: + """ + MkDocs hook to inject markdown content into HTML pages. + + This hook is called after each page is rendered and injects the original + markdown content as JSON for the copy page functionality. + + Args: + output: The HTML output of the page + page: The MkDocs page object + config: The MkDocs configuration + + Returns: + Modified HTML with markdown content injected + """ + return inject_markdown_content(output, page, config) \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 2f8ddad26..745560fbc 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -362,6 +362,7 @@ markdown_extensions: - github-callouts hooks: - _scripts/notebook_hooks.py + - _scripts/copy_page_hooks.py extra: social: - icon: fontawesome/brands/js diff --git a/docs/overrides/copy-page.css b/docs/overrides/copy-page.css new file mode 100644 index 000000000..17360f4ba --- /dev/null +++ b/docs/overrides/copy-page.css @@ -0,0 +1,16 @@ +/* Minimal CSS for copy page button */ +.copy-page-btn { + background: transparent; + border: 1px solid var(--md-default-fg-color--lightest); + padding: 6px 12px; + margin-right: 8px; + border-radius: 4px; + cursor: pointer; + font-size: 14px; + color: var(--md-default-fg-color); + transition: all 0.2s ease; +} + +.copy-page-btn:hover { + background: var(--md-default-fg-color--lightest); +} \ No newline at end of file diff --git a/docs/overrides/copy-page.js b/docs/overrides/copy-page.js new file mode 100644 index 000000000..61afb1900 --- /dev/null +++ b/docs/overrides/copy-page.js @@ -0,0 +1,38 @@ +// Simple copy page functionality - just copy the markdown content +function copyPageAsMarkdown() { + const markdownScript = document.getElementById('page-markdown-content'); + if (!markdownScript) { + alert('Markdown content not available for this page'); + return; + } + + try { + const data = JSON.parse(markdownScript.textContent); + const content = `# ${data.title}\n\nSource: ${window.location.href}\n\n${data.markdown}`; + + navigator.clipboard.writeText(content).then(() => { + // Simple notification + const notification = document.createElement('div'); + notification.textContent = 'Page content copied to clipboard'; + notification.style.cssText = 'position:fixed;top:20px;right:20px;background:#4CAF50;color:white;padding:10px;border-radius:4px;z-index:9999;'; + document.body.appendChild(notification); + setTimeout(() => notification.remove(), 3000); + }).catch(() => { + alert('Failed to copy content'); + }); + } catch (e) { + alert('Failed to parse page content'); + } +} + +// Add button to header - simpler approach +document.addEventListener('DOMContentLoaded', function() { + const headerSource = document.querySelector('.md-header__source'); + if (headerSource) { + const button = document.createElement('button'); + button.textContent = 'Copy page'; + button.onclick = copyPageAsMarkdown; + button.style.cssText = 'background:none;border:1px solid #ddd;padding:6px 12px;margin-right:8px;border-radius:4px;cursor:pointer;'; + headerSource.parentNode.insertBefore(button, headerSource); + } +}); \ No newline at end of file diff --git a/docs/overrides/main.html b/docs/overrides/main.html index b3e63c56d..3e35ab581 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -13,6 +13,130 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= {% block extrahead %} + {% endblock %} diff --git a/docs/package.json b/docs/package.json index 1a5908a84..abcdef39c 100644 --- a/docs/package.json +++ b/docs/package.json @@ -9,7 +9,8 @@ "@langchain/core": "^0.3.38", "@langchain/openai": "^0.4.2", "msgpack-lite": "^0.1.26", - "nock": "^14.0.1" + "nock": "^14.0.1", + "he": "^1.2.0" }, "devDependencies": { "@tsconfig/recommended": "^1.0.8",