chore(docs): notebook convert script + 1 conversion (#3390)

* Adds a conversion script from ipython notebook to markdown.
* Replaces one ipython notebook (create react agent) with a markdown file for testing.

---------

Co-authored-by: Ben Burns <803016+benjamincburns@users.noreply.github.com>

docs/_scripts/assets/vcr_setup_preamble.py

@@ -1,15 +1,12 @@
 import base64
 import os
 import zlib
-from logging import getLogger
 from types import TracebackType
 from typing import Optional, Any, Type
 
 import msgpack
 import vcr
 
-logger = getLogger(__name__)
-
 os.environ.pop("LANGCHAIN_TRACING_V2", None)
 custom_vcr = vcr.VCR()
 
@@ -54,8 +51,10 @@ class HashedCassette:
         self.hash_value: str = hash_value
         self.vcr: vcr.VCR = custom_vcr
         self.cassette_context: Optional[Any] = None
+        self.exited: bool = False
 
     def __enter__(self) -> Any:
+        self.exited: bool = False
         # Get the serializer instance from the VCR instance.
         serializer = self.vcr.serializers[self.vcr.serializer]
         # If the cassette file exists, check its embedded hash.
@@ -65,12 +64,10 @@ class HashedCassette:
             try:
                 cassette_data = serializer.deserialize(content)
             except Exception as e:
-                print(f"Error deserializing cassette, removing file: {e}")
                 os.remove(self.cassette_path)
             else:
                 existing_hash = cassette_data.get("cassette_hash")
                 if existing_hash != self.hash_value:
-                    print("Hash mismatch. Removing outdated cassette.")
                     os.remove(self.cassette_path)
         # Now enter the VCR cassette context.
         self.cassette_context = custom_vcr.use_cassette(
@@ -87,6 +84,9 @@ class HashedCassette:
         exc_val: Optional[BaseException] = None,
         exc_tb: Optional[TracebackType] = None,
     ) -> Optional[bool]:
+        if self.exited:
+            return
+        self.exited = True
         # Exit the VCR cassette context.
         result = self.cassette_context.__exit__(exc_type, exc_val, exc_tb)
         serializer = self.vcr.serializers[self.vcr.serializer]
@@ -97,7 +97,6 @@ class HashedCassette:
             try:
                 cassette_data = serializer.deserialize(content)
             except Exception as e:
-                logger.error(f"Error deserializing cassette during exit: {e}")
                 return result
             # Update the cassette data with the expected hash.
             if cassette_data.get("cassette_hash") != self.hash_value:

docs/_scripts/notebook_convert.py

@@ -1,6 +1,8 @@
+import argparse
 import os
 import re
 from pathlib import Path
+from typing import Literal, Optional
 
 import nbformat
 from nbconvert.exporters import MarkdownExporter
@@ -8,23 +10,45 @@ from nbconvert.preprocessors import Preprocessor
 
 
 class EscapePreprocessor(Preprocessor):
+    def __init__(self, rewrite_links: bool = True, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.rewrite_links = rewrite_links
+
     def preprocess_cell(self, cell, resources, cell_index):
         if cell.cell_type == "markdown":
-            # rewrite markdown links to html links (excluding image links)
-            cell.source = re.sub(
-                r"(?<!!)\[([^\]]*)\]\((?![^\)]*//)([^)]*)(?:\.ipynb)?\)",
-                r'<a href="\2">\1</a>',
-                cell.source,
-            )
+            if self.rewrite_links:
+                # We'll need to adjust the logic for this to keep markdown format
+                # but link to markdown files rather than ipynb files.
+                cell.source = re.sub(
+                    r"(?<!!)\[([^\]]*)\]\((?![^\)]*//)([^)]*)(?:\.ipynb)?\)",
+                    r'<a href="\2">\1</a>',
+                    cell.source,
+                )
+            else:
+                # Keep format but replace the .ipynb extension with .md
+                cell.source = re.sub(
+                    r"(?<!!)\[([^\]]*)\]\((?![^\)]*//)([^)]*)(?:\.ipynb)?\)",
+                    r"[\1](\2.md)",
+                    cell.source,
+                )
+
             # Fix image paths in <img> tags
             cell.source = re.sub(
                 r'<img\s+src="\.?/img/([^"]+)"', r'<img src="../img/\1"', cell.source
             )
 
         elif cell.cell_type == "code":
+            # Determine if the cell has bash or cell magic
+            if cell.source.startswith("%") or cell.source.startswith("!"):
+                # update metadata to denote that it's not a python cell
+                cell.metadata["language_info"] = {"name": "unknown"}
+
             # Remove noqa comments
-            cell.source = re.sub(r'#\s*noqa.*$', '', cell.source, flags=re.MULTILINE)
+            cell.source = re.sub(r"#\s*noqa.*$", "", cell.source, flags=re.MULTILINE)
             # escape ``` in code
+            # This is needed because the markdown exporter will wrap code blocks in
+            # triple backticks, which will break the markdown output if the code block
+            # contains triple backticks.
             cell.source = cell.source.replace("```", r"\`\`\`")
             # escape ``` in output
             if "outputs" in cell:
@@ -114,12 +138,111 @@ exporter = MarkdownExporter(
     ],
 )
 
+md_executable = MarkdownExporter(
+    preprocessors=[
+        ExtractAttachmentsPreprocessor,
+        EscapePreprocessor(rewrite_links=False),
+    ],
+    template_name="md_executable",
+    extra_template_basedirs=[
+        os.path.join(os.path.dirname(__file__), "notebook_convert_templates")
+    ],
+)
+
 
 def convert_notebook(
     notebook_path: Path,
-) -> Path:
+    mode: Literal["markdown", "exec"] = "markdown",
+) -> str:
     with open(notebook_path) as f:
         nb = nbformat.read(f, as_version=4)
 
-    body, _ = exporter.from_notebook_node(nb)
+    nb.metadata.mode = mode
+    if mode == "markdown":
+        body, _ = exporter.from_notebook_node(nb)
+    else:
+        body, _ = md_executable.from_notebook_node(nb)
     return body
+
+
+HERE = Path(__file__).parent
+DOCS = HERE.parent / "docs"
+
+
+# Convert notebooks to markdown
+def _convert_notebooks(
+    *,
+    output_dir: Optional[Path] = None,
+    replace: bool = False,
+    pattern: str = "*.ipynb",
+) -> None:
+    """Converting notebooks."""
+    if not output_dir and not replace:
+        raise ValueError("Either --output_dir or --replace must be specified")
+
+    output_dir_path = DOCS if replace else Path(output_dir)
+    notebooks = list(DOCS.rglob(pattern))
+
+    file_names = [notebook.name for notebook in notebooks]
+
+    for notebook in notebooks:
+        markdown = convert_notebook(notebook, mode="exec")
+        markdown_path = output_dir_path / notebook.relative_to(DOCS).with_suffix(".md")
+        markdown_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(markdown_path, "w") as f:
+            f.write(markdown)
+        if replace:
+            notebook.unlink(missing_ok=False)
+
+    if replace:
+        # The regex will match markdown links that point to *.ipynb files.
+        # It captures:
+        #   group(1): the link text (inside the square brackets)
+        #   group(2): the file path (without the trailing .ipynb)
+        link_pattern = r"(?<!!)\[([^\]]+)\]\((?![^)]*//)([^)]+)\.ipynb\)"
+
+        def replace_link(match: re.Match) -> str:
+            link_text = match.group(1)
+            link_target = match.group(2)
+            # Reconstruct the file name with the .ipynb extension.
+            # For example, if link_target is "foo/bar", then linked_file becomes "bar.ipynb".
+            linked_file = Path(link_target).name + ".ipynb"
+            # Only update if the notebook was among those converted.
+            if linked_file in file_names:
+                # Change the extension from .ipynb to .md
+                return f"[{link_text}]({link_target}.md)"
+            # Otherwise, leave the original link intact.
+            return match.group(0)
+
+        # Process all markdown files in the output directory.
+        for path in output_dir_path.rglob("*.md"):
+            with open(path, "r", encoding="utf-8") as f:
+                content = f.read()
+            new_content = re.sub(link_pattern, replace_link, content)
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(new_content)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Convert notebooks to markdown")
+    parser.add_argument(
+        "--output_dir",
+        default=None,
+        help="Directory to output markdown files",
+    )
+    parser.add_argument(
+        "--replace",
+        action="store_true",
+        help="Replace original notebooks with markdown files",
+    )
+    parser.add_argument(
+        "--pattern",
+        default="*.ipynb",
+        help="Glob pattern to match notebooks to convert",
+    )
+    args = parser.parse_args()
+    _convert_notebooks(
+        replace=args.replace,
+        output_dir=args.output_dir,
+        pattern=args.pattern,
+    )

docs/_scripts/notebook_convert_templates/md_executable/conf.json

@@ -0,0 +1,5 @@
+{
+  "mimetypes": {
+    "text/markdown": true
+  }
+}

docs/_scripts/notebook_convert_templates/md_executable/index.md.j2

@@ -0,0 +1,36 @@
+{#https://github.com/rdbisme/nbconvert/blob/master/share/jupyter/nbconvert/templates/markdown/index.md.j2#}
+{% extends 'markdown/index.md.j2' %}
+
+{% block input %}
+```
+{%- if 'magics_language' in cell.metadata  -%}
+    {{ cell.metadata.magics_language}}
+{%- elif 'name' in nb.metadata.get('language_info', {}) -%}
+    {{ nb.metadata.language_info.name }} exec="on" source="above" session="1"
+{%- endif %}
+{{ cell.source}}
+```
+{% endblock input %}
+
+{%- block traceback_line -%}
+{%- endblock traceback_line -%}
+
+{%- block stream -%}
+{%- endblock stream -%}
+
+{%- block data_text scoped -%}
+{%- endblock data_text -%}
+
+{%- block data_html scoped -%}
+```html
+{{ output.data['text/html'] | safe }} 
+```
+{%- endblock data_html -%}
+
+{%- block data_jpg scoped -%}
+![](data:image/jpg;base64,{{ output.data['image/jpeg'] }})
+{%- endblock data_jpg -%}
+
+{%- block data_png scoped -%}
+![](data:image/png;base64,{{ output.data['image/png'] }})
+{%- endblock data_png -%}

docs/_scripts/notebook_hooks.py

@@ -16,6 +16,7 @@ from generate_api_reference_links import update_markdown_with_imports
 from notebook_convert import convert_notebook
 from setup_vcr import load_postamble, load_preamble, _hash_string
 
+
 logger = logging.getLogger(__name__)
 logging.basicConfig()
 logger.setLevel(logging.INFO)
@@ -163,26 +164,27 @@ def handle_vcr_setup(
     id: str,
     md: Markdown,
     **kwargs: Dict[str, Any],
-) -> str:
+) -> Dict[str, Any]:
     """Handle VCR setup in markdown content if necessary."""
     try:
         if kwargs.get("extra", None) is None:
-            raise ValueError(
+            raise SuperFencesException(
                 f"error while processing {language} block: extra dict is required"
             )
 
         if kwargs["extra"].get("path", None) is None:
-            raise ValueError(
+            raise SuperFencesException(
                 f"error while processing {language} block: path is required"
             )
 
         document_filename = kwargs["extra"]["path"]
 
-        logger.info("document_filename: %s", document_filename)
-
         if session is None or session == "" and id is None or id == "":
             id = _hash_string(code)
 
+        if session is not None and session != "":
+            logger.info(f"new session {session} on page {document_filename}")
+
         cassette_prefix = document_filename.replace(".md", "").replace(os.path.sep, "_")
 
         cassette_dir = os.path.abspath(
@@ -204,6 +206,9 @@ def handle_vcr_setup(
         ]
 
         if session is None or session == "":
+            logger.info(
+                f"no session, adding postamble for {language} in {document_filename}"
+            )
             wrapped_lines.append(load_postamble(language))
 
         transformed_source = "\n".join(wrapped_lines)
@@ -223,20 +228,34 @@ def handle_vcr_teardown(
     session: str,
     history: list[SessionHistoryEntry],
 ):
-    session = history[-1].inputs["session"]
-    inputs = dict(history[-1].inputs)
-    del inputs["session"]
-    del inputs["code"]
-    del inputs["language"]
-    del inputs["id"]
-    formatter(
-        code="_cassette.__exit__() # markdown-exec: hide",
-        language="python",
+    last_inputs = dict(history[-1].inputs)
+    code = load_postamble(language)
+    md = last_inputs["md"]
+    html = False
+    update_toc = False
+
+    document_filename = last_inputs.get("extra", {}).get("path", None)
+
+    if document_filename is None:
+        logger.warning(f"no document filename found while tearing down {session}!")
+    else:
+        logger.info(f"tearing down {session} on {document_filename}")
+        logger.info(traceback.format_stack())
+
+    kwargs = dict(
+        code=code,
         session=session,
         id=f"{id}_vcr_end",
-        **inputs,
+        md=md,
+        html=html,
+        update_toc=update_toc,
+        extra={},
     )
 
+    # This doesn't actually render anything, we just call the formatter so it
+    # executes in the same context as the session of which we're disposing.
+    formatter(**kwargs)
+
 
 def _on_page_markdown_with_config(
     markdown: str,
@@ -250,7 +269,7 @@ def _on_page_markdown_with_config(
         return markdown
 
     if page.file.src_path.endswith(".ipynb"):
-        logger.info("Processing Jupyter notebook: %s", page.file.src_path)
+        # logger.info("Processing Jupyter notebook: %s", page.file.src_path)
         markdown = convert_notebook(page.file.abs_src_path)
 
     # Append API reference links to code blocks
@@ -266,7 +285,7 @@ def _on_page_markdown_with_config(
 
     if remove_base64_images:
         # Remove base64 encoded images from markdown
-        markdown = re.sub(r"!\[.*?\]\(data:image/[^;]+;base64,[^\)]+\)", "", markdown)
+        markdown = re.sub(r"!\[.*?\]\(data:image/+;base64,[^\)]+\)", "", markdown)
 
     return markdown
 

docs/_scripts/setup_vcr.py

@@ -37,12 +37,16 @@ def _get_typescript_cassette_cleanup() -> str:
 
 preamble_inits = {
     "python": _get_python_cassette_init,
+    "py": _get_python_cassette_init,
     "typescript": _get_typescript_cassette_init,
+    "ts": _get_typescript_cassette_init,
 }
 
 preamble_cleanups = {
     "python": _get_python_cassette_cleanup,
+    "py": _get_python_cassette_cleanup,
     "typescript": _get_typescript_cassette_cleanup,
+    "ts": _get_typescript_cassette_cleanup,
 }