docs: cell magic to shell block conversion (#3433)

* add handling for blocks that use magic commands like %pip to convert
them into bash
* Apply new logic to another notebook

docs/_scripts/notebook_convert.py

@@ -27,16 +27,52 @@ def _uses_input(source: str) -> bool:
     return False
 
 
+def _rewrite_cell_magic(code: str) -> str:
+    """Process a code block that uses cell magic.:w
+
+    - Lines starting with "%%capture" are ignored.
+    - Lines starting with "%pip" are rewritten by removing the leading "%" character.
+    - Any other non-empty line causes a NotImplementedError.
+
+    Args:
+        code (str): The original code block.
+
+    Returns:
+        str: The transformed code block.
+
+    Raises:
+        NotImplementedError: If a line doesn't start with either "%%capture" or "%pip".
+    """
+    rewritten_lines = []
+
+    for line in code.splitlines():
+        stripped = line.strip()
+        # Skip empty lines
+        if not stripped:
+            continue
+        # Ignore %%capture lines
+        if stripped.startswith("%%capture"):
+            continue
+        # Rewrite %pip lines by dropping the '%'
+        elif stripped.startswith("%pip"):
+            # Drop the leading '%' character
+            rewritten_lines.append(stripped[1:])
+        # Anything else is not supported
+        else:
+            raise NotImplementedError(f"Unhandled line: {line}")
+
+    return "\n".join(rewritten_lines)
+
+
 class EscapePreprocessor(Preprocessor):
-    def __init__(self, rewrite_links: bool = True, **kwargs) -> None:
+    def __init__(self, markdown_exec_migration: bool = False, **kwargs) -> None:
         super().__init__(**kwargs)
-        self.rewrite_links = rewrite_links
+        self.markdown_exec_migration = markdown_exec_migration
 
     def preprocess_cell(self, cell, resources, cell_index):
         if cell.cell_type == "markdown":
-            if self.rewrite_links:
-                # We'll need to adjust the logic for this to keep markdown format
-                # but link to markdown files rather than ipynb files.
+            if not self.markdown_exec_migration:
+                # Old logic is to convert ipynb links to HTML links
                 cell.source = re.sub(
                     r"(?<!!)\[([^\]]*)\]\((?![^\)]*//)([^)]*)(?:\.ipynb)?\)",
                     r'<a href="\2">\1</a>',
@@ -110,6 +146,12 @@ class EscapePreprocessor(Preprocessor):
             )
             cell.metadata["exec"] = is_exec
 
+            if self.markdown_exec_migration:
+                # For markdown exec migration we'll re-write cell magic as bash commands
+                if source.startswith("%%"):
+                    cell.source = _rewrite_cell_magic(source)
+                    cell.metadata["language"] = "shell"
+
             # Remove noqa comments
             cell.source = re.sub(r"#\s*noqa.*$", "", cell.source, flags=re.MULTILINE)
             # escape ``` in code
@@ -208,7 +250,7 @@ exporter = MarkdownExporter(
 md_executable = MarkdownExporter(
     preprocessors=[
         ExtractAttachmentsPreprocessor,
-        EscapePreprocessor(rewrite_links=False),
+        EscapePreprocessor(markdown_exec_migration=True),
     ],
     template_name="md_executable",
     extra_template_basedirs=[

docs/_scripts/notebook_convert_templates/md_executable/index.md.j2

@@ -5,6 +5,8 @@
 ```
 {%- if 'magics_language' in cell.metadata  -%}
     {{ cell.metadata.magics_language}}
+{%- elif cell.metadata.get('language') == "shell" -%}
+    shell
 {%- elif 'name' in nb.metadata.get('language_info', {}) -%}
     {{ nb.metadata.language_info.name }}{% if cell.metadata.exec|default(false) %} exec="on" source="above" session="1" result="ansi"{% endif %}
 {%- endif %}

docs/docs/how-tos/create-react-agent.md

@@ -47,7 +47,7 @@ First let's install the required packages and set our API keys
 ```
 
 
-```python exec="on" source="above" session="1"
+```python
 import getpass
 import os
 

docs/docs/how-tos/index.md

@@ -13,7 +13,7 @@ Here you’ll find answers to “How do I...?” types of questions. These guide
 
 - [How to update graph state from nodes](state-reducers.md)
 - [How to create a sequence of steps](sequence.md)
-- [How to create branches for parallel execution](branching.ipynb)
+- [How to create branches for parallel execution](branching.md)
 - [How to create and control loops with recursion limits](recursion-limit.ipynb)
 - [How to visualize your graph](visualization.ipynb)
 

docs/mkdocs.yml

@@ -125,7 +125,7 @@ nav:
               - Graph API Basics: how-tos#graph-api-basics
               - how-tos/state-reducers.md
               - how-tos/sequence.md
-              - how-tos/branching.ipynb
+              - how-tos/branching.md
               - how-tos/recursion-limit.ipynb
               - how-tos/visualization.ipynb
           - Controllability:

docs/tests/unit_tests/test_notebook_conversion.py

@@ -21,13 +21,12 @@ def test_convert_normal_code_block() -> None:
 # We treat cell magic as a non-executable code block.
 CELL_MAGIC_INPUT = """\
 %%capture
-print("Hello, world!")\
+%pip install numpy
 """
 
 CELL_MAGIC_OUTPUT = """\
-```python
-%%capture
-print("Hello, world!")
+```shell
+pip install numpy
 ```
 """