chore(docs): notebook convert script + 1 conversion (#3390)

* Adds a conversion script from ipython notebook to markdown.
* Replaces one ipython notebook (create react agent) with a markdown file for testing.

---------

Co-authored-by: Ben Burns <803016+benjamincburns@users.noreply.github.com>

.github/workflows/codespell.yml

@@ -36,7 +36,7 @@
           - name: Codespell
             uses: codespell-project/actions-codespell@v2
             with:
-              skip: '*.ambr,*.lock,*.ipynb,*.yaml,*.zlib'
+              skip: '*.ambr,*.lock,*.ipynb,*.yaml,*.zlib,*.md'
               ignore_words_list: ${{ steps.extract_ignore_words.outputs.ignore_words_list }}
           # We do this to avoid spellchecking cell outputs
           - name: Codespell Notebooks

.github/workflows/deploy_docs.yml

@@ -48,7 +48,7 @@ jobs:
   deploy:
     # needs: run-changed-notebooks
     runs-on: ubuntu-latest
-    timeout-minutes: 10  # Job will be cancelled if it runs for more than 10 minutes
+    timeout-minutes: 10 # Job will be cancelled if it runs for more than 10 minutes
     env:
       GITHUB_TOKEN: ${{ secrets.MKDOCS_GITHUB_TOKEN }}
     steps:
@@ -63,17 +63,29 @@ jobs:
           poetry-version: ${{ env.POETRY_VERSION }}
           cache-key: docs
 
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "22"
+          cache: "yarn"
+          cache-dependency-path: docs/yarn.lock
+
       - name: Install dependencies
         run: |
-          poetry install --with test --no-root
+          yarn
+          poetry install --with test --with docs --no-root
           poetry run pip install -U \
             pytest \
             pytest-check-links \
-            langsmith \
-            langchain \
             GitPython \
             "git+https://${GITHUB_TOKEN}@github.com/langchain-ai/mkdocs-material-insiders.git" \
-            "git+https://github.com/benjamincburns/markdown-exec.git@10cdd338bfdb1f99705b3a4d06b244f7f185ecae"
+            "git+https://github.com/benjamincburns/markdown-exec.git@cc0d39d737e5ffd4b83d23cd8729d7ea16e363c8"
+
+          poetry run jupyter kernelspec list
+          poetry run python3 -m ipykernel install --user --name=python3
+          npm install -g tslab
+          poetry run tslab install --python=python3
+          poetry run jupyter kernelspec list
 
       - name: Lint Docs
         # This step lints the docs using the existing linting set up.
@@ -85,6 +97,8 @@ jobs:
         run: make build-docs
         env:
           MKDOCS_GIT_COMMITTERS_APIKEY: ${{ secrets.MKDOCS_GIT_COMMITTERS_APIKEY }}
+          OPENAI_API_KEY: sf-proj-1234567890 # fake placeholder, shouldn't actually be used
+          ANTHROPIC_API_KEY: sk-ant-api03-1234567890 # fake placeholder, shouldn't actually be used
       - name: Check links in notebooks
         env:
           LANGCHAIN_API_KEY: test
@@ -135,7 +149,7 @@ jobs:
         uses: actions/configure-pages@v4
 
       - name: Upload Pages Artifact
-        if: github.ref == 'refs/heads/main'
+        # if: github.ref == 'refs/heads/main'
         uses: actions/upload-pages-artifact@v3
         with:
           path: ./docs/site/

docs/.gitignore

@@ -2,4 +2,3 @@ site/
 docs/cloud/reference/sdk/js_ts_sdk_ref.md
 
 .vercel
-cassettes/

docs/Makefile

@@ -20,12 +20,20 @@ llms-text:
 	poetry run python _scripts/generate_llms_text.py docs/llms-full.txt
 
 install-vercel-deps:
+	dnf install -y python3.11
 	curl -sSL https://install.python-poetry.org | python3 -
 	poetry self update 1.8.5
+	# don't use vercel's python - it wasn't compiled with sqlite support, and it fails when installing ipython's kernel
+	poetry env use /usr/bin/python3.11
+	poetry install --with docs --with test --no-root
+	poetry run pip install "git+https://github.com/benjamincburns/markdown-exec.git@cc0d39d737e5ffd4b83d23cd8729d7ea16e363c8"
+	poetry run python3 -m ipykernel install --name=python3
+	npm install -g tslab
+	poetry run tslab install --python=python3
+	poetry run jupyter kernelspec list
+
 
 vercel-build-docs: install-vercel-deps
-	poetry install
-	poetry run pip install "git+https://github.com/benjamincburns/markdown-exec.git@26c64551340da6ffcc8cb4f53db99e8c239d9919"
 	make build-docs
 
 

docs/_scripts/assets/vcr_setup_preamble.py

@@ -1,15 +1,12 @@
 import base64
 import os
 import zlib
-from logging import getLogger
 from types import TracebackType
 from typing import Optional, Any, Type
 
 import msgpack
 import vcr
 
-logger = getLogger(__name__)
-
 os.environ.pop("LANGCHAIN_TRACING_V2", None)
 custom_vcr = vcr.VCR()
 
@@ -54,8 +51,10 @@ class HashedCassette:
         self.hash_value: str = hash_value
         self.vcr: vcr.VCR = custom_vcr
         self.cassette_context: Optional[Any] = None
+        self.exited: bool = False
 
     def __enter__(self) -> Any:
+        self.exited: bool = False
         # Get the serializer instance from the VCR instance.
         serializer = self.vcr.serializers[self.vcr.serializer]
         # If the cassette file exists, check its embedded hash.
@@ -65,12 +64,10 @@ class HashedCassette:
             try:
                 cassette_data = serializer.deserialize(content)
             except Exception as e:
-                print(f"Error deserializing cassette, removing file: {e}")
                 os.remove(self.cassette_path)
             else:
                 existing_hash = cassette_data.get("cassette_hash")
                 if existing_hash != self.hash_value:
-                    print("Hash mismatch. Removing outdated cassette.")
                     os.remove(self.cassette_path)
         # Now enter the VCR cassette context.
         self.cassette_context = custom_vcr.use_cassette(
@@ -87,6 +84,9 @@ class HashedCassette:
         exc_val: Optional[BaseException] = None,
         exc_tb: Optional[TracebackType] = None,
     ) -> Optional[bool]:
+        if self.exited:
+            return
+        self.exited = True
         # Exit the VCR cassette context.
         result = self.cassette_context.__exit__(exc_type, exc_val, exc_tb)
         serializer = self.vcr.serializers[self.vcr.serializer]
@@ -97,7 +97,6 @@ class HashedCassette:
             try:
                 cassette_data = serializer.deserialize(content)
             except Exception as e:
-                logger.error(f"Error deserializing cassette during exit: {e}")
                 return result
             # Update the cassette data with the expected hash.
             if cassette_data.get("cassette_hash") != self.hash_value:

docs/_scripts/notebook_convert.py

@@ -1,6 +1,8 @@
+import argparse
 import os
 import re
 from pathlib import Path
+from typing import Literal, Optional
 
 import nbformat
 from nbconvert.exporters import MarkdownExporter
@@ -8,23 +10,45 @@ from nbconvert.preprocessors import Preprocessor
 
 
 class EscapePreprocessor(Preprocessor):
+    def __init__(self, rewrite_links: bool = True, **kwargs) -> None:
+        super().__init__(**kwargs)
+        self.rewrite_links = rewrite_links
+
     def preprocess_cell(self, cell, resources, cell_index):
         if cell.cell_type == "markdown":
-            # rewrite markdown links to html links (excluding image links)
-            cell.source = re.sub(
-                r"(?<!!)\[([^\]]*)\]\((?![^\)]*//)([^)]*)(?:\.ipynb)?\)",
-                r'<a href="\2">\1</a>',
-                cell.source,
-            )
+            if self.rewrite_links:
+                # We'll need to adjust the logic for this to keep markdown format
+                # but link to markdown files rather than ipynb files.
+                cell.source = re.sub(
+                    r"(?<!!)\[([^\]]*)\]\((?![^\)]*//)([^)]*)(?:\.ipynb)?\)",
+                    r'<a href="\2">\1</a>',
+                    cell.source,
+                )
+            else:
+                # Keep format but replace the .ipynb extension with .md
+                cell.source = re.sub(
+                    r"(?<!!)\[([^\]]*)\]\((?![^\)]*//)([^)]*)(?:\.ipynb)?\)",
+                    r"[\1](\2.md)",
+                    cell.source,
+                )
+
             # Fix image paths in <img> tags
             cell.source = re.sub(
                 r'<img\s+src="\.?/img/([^"]+)"', r'<img src="../img/\1"', cell.source
             )
 
         elif cell.cell_type == "code":
+            # Determine if the cell has bash or cell magic
+            if cell.source.startswith("%") or cell.source.startswith("!"):
+                # update metadata to denote that it's not a python cell
+                cell.metadata["language_info"] = {"name": "unknown"}
+
             # Remove noqa comments
-            cell.source = re.sub(r'#\s*noqa.*$', '', cell.source, flags=re.MULTILINE)
+            cell.source = re.sub(r"#\s*noqa.*$", "", cell.source, flags=re.MULTILINE)
             # escape ``` in code
+            # This is needed because the markdown exporter will wrap code blocks in
+            # triple backticks, which will break the markdown output if the code block
+            # contains triple backticks.
             cell.source = cell.source.replace("```", r"\`\`\`")
             # escape ``` in output
             if "outputs" in cell:
@@ -114,12 +138,111 @@ exporter = MarkdownExporter(
     ],
 )
 
+md_executable = MarkdownExporter(
+    preprocessors=[
+        ExtractAttachmentsPreprocessor,
+        EscapePreprocessor(rewrite_links=False),
+    ],
+    template_name="md_executable",
+    extra_template_basedirs=[
+        os.path.join(os.path.dirname(__file__), "notebook_convert_templates")
+    ],
+)
+
 
 def convert_notebook(
     notebook_path: Path,
-) -> Path:
+    mode: Literal["markdown", "exec"] = "markdown",
+) -> str:
     with open(notebook_path) as f:
         nb = nbformat.read(f, as_version=4)
 
-    body, _ = exporter.from_notebook_node(nb)
+    nb.metadata.mode = mode
+    if mode == "markdown":
+        body, _ = exporter.from_notebook_node(nb)
+    else:
+        body, _ = md_executable.from_notebook_node(nb)
     return body
+
+
+HERE = Path(__file__).parent
+DOCS = HERE.parent / "docs"
+
+
+# Convert notebooks to markdown
+def _convert_notebooks(
+    *,
+    output_dir: Optional[Path] = None,
+    replace: bool = False,
+    pattern: str = "*.ipynb",
+) -> None:
+    """Converting notebooks."""
+    if not output_dir and not replace:
+        raise ValueError("Either --output_dir or --replace must be specified")
+
+    output_dir_path = DOCS if replace else Path(output_dir)
+    notebooks = list(DOCS.rglob(pattern))
+
+    file_names = [notebook.name for notebook in notebooks]
+
+    for notebook in notebooks:
+        markdown = convert_notebook(notebook, mode="exec")
+        markdown_path = output_dir_path / notebook.relative_to(DOCS).with_suffix(".md")
+        markdown_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(markdown_path, "w") as f:
+            f.write(markdown)
+        if replace:
+            notebook.unlink(missing_ok=False)
+
+    if replace:
+        # The regex will match markdown links that point to *.ipynb files.
+        # It captures:
+        #   group(1): the link text (inside the square brackets)
+        #   group(2): the file path (without the trailing .ipynb)
+        link_pattern = r"(?<!!)\[([^\]]+)\]\((?![^)]*//)([^)]+)\.ipynb\)"
+
+        def replace_link(match: re.Match) -> str:
+            link_text = match.group(1)
+            link_target = match.group(2)
+            # Reconstruct the file name with the .ipynb extension.
+            # For example, if link_target is "foo/bar", then linked_file becomes "bar.ipynb".
+            linked_file = Path(link_target).name + ".ipynb"
+            # Only update if the notebook was among those converted.
+            if linked_file in file_names:
+                # Change the extension from .ipynb to .md
+                return f"[{link_text}]({link_target}.md)"
+            # Otherwise, leave the original link intact.
+            return match.group(0)
+
+        # Process all markdown files in the output directory.
+        for path in output_dir_path.rglob("*.md"):
+            with open(path, "r", encoding="utf-8") as f:
+                content = f.read()
+            new_content = re.sub(link_pattern, replace_link, content)
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(new_content)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Convert notebooks to markdown")
+    parser.add_argument(
+        "--output_dir",
+        default=None,
+        help="Directory to output markdown files",
+    )
+    parser.add_argument(
+        "--replace",
+        action="store_true",
+        help="Replace original notebooks with markdown files",
+    )
+    parser.add_argument(
+        "--pattern",
+        default="*.ipynb",
+        help="Glob pattern to match notebooks to convert",
+    )
+    args = parser.parse_args()
+    _convert_notebooks(
+        replace=args.replace,
+        output_dir=args.output_dir,
+        pattern=args.pattern,
+    )

docs/_scripts/notebook_convert_templates/md_executable/conf.json

@@ -0,0 +1,5 @@
+{
+  "mimetypes": {
+    "text/markdown": true
+  }
+}

docs/_scripts/notebook_convert_templates/md_executable/index.md.j2

@@ -0,0 +1,36 @@
+{#https://github.com/rdbisme/nbconvert/blob/master/share/jupyter/nbconvert/templates/markdown/index.md.j2#}
+{% extends 'markdown/index.md.j2' %}
+
+{% block input %}
+```
+{%- if 'magics_language' in cell.metadata  -%}
+    {{ cell.metadata.magics_language}}
+{%- elif 'name' in nb.metadata.get('language_info', {}) -%}
+    {{ nb.metadata.language_info.name }} exec="on" source="above" session="1"
+{%- endif %}
+{{ cell.source}}
+```
+{% endblock input %}
+
+{%- block traceback_line -%}
+{%- endblock traceback_line -%}
+
+{%- block stream -%}
+{%- endblock stream -%}
+
+{%- block data_text scoped -%}
+{%- endblock data_text -%}
+
+{%- block data_html scoped -%}
+```html
+{{ output.data['text/html'] | safe }} 
+```
+{%- endblock data_html -%}
+
+{%- block data_jpg scoped -%}
+![](data:image/jpg;base64,{{ output.data['image/jpeg'] }})
+{%- endblock data_jpg -%}
+
+{%- block data_png scoped -%}
+![](data:image/png;base64,{{ output.data['image/png'] }})
+{%- endblock data_png -%}

docs/_scripts/notebook_hooks.py

@@ -16,6 +16,7 @@ from generate_api_reference_links import update_markdown_with_imports
 from notebook_convert import convert_notebook
 from setup_vcr import load_postamble, load_preamble, _hash_string
 
+
 logger = logging.getLogger(__name__)
 logging.basicConfig()
 logger.setLevel(logging.INFO)
@@ -163,26 +164,27 @@ def handle_vcr_setup(
     id: str,
     md: Markdown,
     **kwargs: Dict[str, Any],
-) -> str:
+) -> Dict[str, Any]:
     """Handle VCR setup in markdown content if necessary."""
     try:
         if kwargs.get("extra", None) is None:
-            raise ValueError(
+            raise SuperFencesException(
                 f"error while processing {language} block: extra dict is required"
             )
 
         if kwargs["extra"].get("path", None) is None:
-            raise ValueError(
+            raise SuperFencesException(
                 f"error while processing {language} block: path is required"
             )
 
         document_filename = kwargs["extra"]["path"]
 
-        logger.info("document_filename: %s", document_filename)
-
         if session is None or session == "" and id is None or id == "":
             id = _hash_string(code)
 
+        if session is not None and session != "":
+            logger.info(f"new session {session} on page {document_filename}")
+
         cassette_prefix = document_filename.replace(".md", "").replace(os.path.sep, "_")
 
         cassette_dir = os.path.abspath(
@@ -204,6 +206,9 @@ def handle_vcr_setup(
         ]
 
         if session is None or session == "":
+            logger.info(
+                f"no session, adding postamble for {language} in {document_filename}"
+            )
             wrapped_lines.append(load_postamble(language))
 
         transformed_source = "\n".join(wrapped_lines)
@@ -223,20 +228,34 @@ def handle_vcr_teardown(
     session: str,
     history: list[SessionHistoryEntry],
 ):
-    session = history[-1].inputs["session"]
-    inputs = dict(history[-1].inputs)
-    del inputs["session"]
-    del inputs["code"]
-    del inputs["language"]
-    del inputs["id"]
-    formatter(
-        code="_cassette.__exit__() # markdown-exec: hide",
-        language="python",
+    last_inputs = dict(history[-1].inputs)
+    code = load_postamble(language)
+    md = last_inputs["md"]
+    html = False
+    update_toc = False
+
+    document_filename = last_inputs.get("extra", {}).get("path", None)
+
+    if document_filename is None:
+        logger.warning(f"no document filename found while tearing down {session}!")
+    else:
+        logger.info(f"tearing down {session} on {document_filename}")
+        logger.info(traceback.format_stack())
+
+    kwargs = dict(
+        code=code,
         session=session,
         id=f"{id}_vcr_end",
-        **inputs,
+        md=md,
+        html=html,
+        update_toc=update_toc,
+        extra={},
     )
 
+    # This doesn't actually render anything, we just call the formatter so it
+    # executes in the same context as the session of which we're disposing.
+    formatter(**kwargs)
+
 
 def _on_page_markdown_with_config(
     markdown: str,
@@ -250,7 +269,7 @@ def _on_page_markdown_with_config(
         return markdown
 
     if page.file.src_path.endswith(".ipynb"):
-        logger.info("Processing Jupyter notebook: %s", page.file.src_path)
+        # logger.info("Processing Jupyter notebook: %s", page.file.src_path)
         markdown = convert_notebook(page.file.abs_src_path)
 
     # Append API reference links to code blocks
@@ -266,7 +285,7 @@ def _on_page_markdown_with_config(
 
     if remove_base64_images:
         # Remove base64 encoded images from markdown
-        markdown = re.sub(r"!\[.*?\]\(data:image/[^;]+;base64,[^\)]+\)", "", markdown)
+        markdown = re.sub(r"!\[.*?\]\(data:image/+;base64,[^\)]+\)", "", markdown)
 
     return markdown
 

docs/_scripts/setup_vcr.py

@@ -37,12 +37,16 @@ def _get_typescript_cassette_cleanup() -> str:
 
 preamble_inits = {
     "python": _get_python_cassette_init,
+    "py": _get_python_cassette_init,
     "typescript": _get_typescript_cassette_init,
+    "ts": _get_typescript_cassette_init,
 }
 
 preamble_cleanups = {
     "python": _get_python_cassette_cleanup,
+    "py": _get_python_cassette_cleanup,
     "typescript": _get_typescript_cassette_cleanup,
+    "ts": _get_typescript_cassette_cleanup,
 }
 
 

docs/codespell_notebooks.sh

@@ -1,6 +1,17 @@
 ERROR_FOUND=0
 for file in $(find $1 -name "*.ipynb" | grep -v ".ipynb_checkpoints"); do
-    OUTPUT=$(cat "$file" | jupytext --from ipynb --to py:percent | codespell -)
+    # Adding regexp to ignore base64 strings
+    OUTPUT=$(cat "$file" | jupytext --from ipynb --to py:percent | codespell --ignore-regex='[A-Za-z0-9+/=]{25,}' -)
+    if [ -n "$OUTPUT" ]; then
+        echo "Errors found in $file"
+        echo "$OUTPUT"
+        ERROR_FOUND=1
+    fi
+done
+
+for file in $(find $1 -name "*.md"); do
+    # Adding regexp to ignore base64 strings
+    OUTPUT=$(cat "$file" | codespell --ignore-regex='[A-Za-z0-9+/=]{25,}' -)
     if [ -n "$OUTPUT" ]; then
         echo "Errors found in $file"
         echo "$OUTPUT"

docs/docs/how-tos/index.md

@@ -162,7 +162,7 @@ One of the big benefits of LangGraph is that you can easily create your own agen
 
 These guides show how to use the prebuilt ReAct agent:
 
-- [How to use the pre-built ReAct agent](create-react-agent.ipynb)
+- [How to use the pre-built ReAct agent](create-react-agent.md)
 - [How to add thread-level memory to a ReAct Agent](create-react-agent-memory.ipynb)
 - [How to add a custom system prompt to a ReAct agent](create-react-agent-system-prompt.ipynb)
 - [How to add human-in-the-loop processes to a ReAct agent](create-react-agent-hitl.ipynb)

docs/mkdocs.yml

@@ -203,7 +203,7 @@ nav:
               - how-tos/autogen-integration-functional.ipynb
           - Prebuilt ReAct Agent:
               - Prebuilt ReAct Agent: how-tos#prebuilt-react-agent
-              - how-tos/create-react-agent.ipynb
+              - how-tos/create-react-agent.md
               - how-tos/create-react-agent-memory.ipynb
               - how-tos/create-react-agent-system-prompt.ipynb
               - how-tos/create-react-agent-hitl.ipynb

docs/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "license": "MIT",
   "scripts": {
-    "build": "echo 'export PATH=$PATH:/vercel/.local/bin:$PATH' > ~/.bashrc && source ~/.bashrc && make vercel-build-docs"
+    "build": "echo 'export OPENAI_API_KEY=\"sk-proj-1234567890\"' >> ~/.bashrc && echo 'export ANTHROPIC_API_KEY=\"sk-ant-api03-1234567890\"' >> ~/.bashrc && echo 'export PATH=$PATH:/vercel/.local/bin:$PATH' >> ~/.bashrc && source ~/.bashrc && make vercel-build-docs"
   },
   "dependencies": {
     "@langchain/core": "^0.3.38",