docs: process cell magics (#4462)

Handle a small thing that can be fairly confusing to new python users.

Before:


![image](https://github.com/user-attachments/assets/39011f0c-0a7e-4f32-94d7-a40f0b14f2ab)


After:


![image](https://github.com/user-attachments/assets/16aa4429-eb4e-44b5-8714-29279d93c7ea)

docs/_scripts/notebook_convert.py

@@ -1,7 +1,6 @@
 import ast
 import os
 import re
-from pathlib import Path
 from typing import Literal
 
 import nbformat
@@ -26,7 +25,7 @@ def _uses_input(source: str) -> bool:
 
 
 def _rewrite_cell_magic(code: str) -> str:
-    """Process a code block that uses cell magic.:w
+    """Process a code block that uses cell magic.
 
     - Lines starting with "%%capture" are ignored.
     - Lines starting with "%pip" are rewritten by removing the leading "%" character.
@@ -52,10 +51,14 @@ def _rewrite_cell_magic(code: str) -> str:
         if stripped.startswith("%%capture"):
             continue
         # Rewrite %pip lines by dropping the '%'
-        elif stripped.startswith("%pip"):
-            # Drop the leading '%' character
-            rewritten_lines.append(stripped[1:])
-        # Anything else is not supported
+        elif stripped.startswith("%") or stripped.startswith("!"):
+            # Drop the leading '%' character and then drop all leading whitespace
+            stripped = stripped.lstrip("%! \t")
+            # Check if the line starts with "pip"
+            if stripped.startswith("pip"):
+                rewritten_lines.append(stripped)
+            else:
+                raise NotImplementedError(f"Unhandled line: {line}")
         else:
             raise NotImplementedError(f"Unhandled line: {line}")
 
@@ -247,13 +250,10 @@ class EscapePreprocessor(Preprocessor):
             )
             cell.metadata["exec"] = is_exec
 
-            if self.markdown_exec_migration:
-                # For markdown exec migration we'll re-write cell magic as bash commands
-                if source.startswith("%%"):
-                    cell.source = _rewrite_cell_magic(source)
-                    cell.metadata["language"] = "shell"
-
-                cell.metadata["has_output"] = _has_output(source)
+            # For markdown exec migration we'll re-write cell magic as bash commands
+            if source.startswith("%%"):
+                cell.source = _rewrite_cell_magic(source)
+                cell.metadata["language"] = "shell"
 
             # Remove noqa comments
             cell.source = re.sub(r"#\s*noqa.*$", "", cell.source, flags=re.MULTILINE)
@@ -352,7 +352,7 @@ exporter = MarkdownExporter(
 
 
 def convert_notebook(
-    notebook_path: Path,
+    notebook_path: str,
     mode: Literal["markdown", "exec"] = "markdown",
 ) -> str:
     with open(notebook_path) as f:

docs/_scripts/notebook_convert_templates/mdoutput/index.md.j2

@@ -1,5 +1,18 @@
 {% extends 'markdown/index.md.j2' %}
 
+{% block input %}{# cell.metadata.language is an addition of our docs pipeline. #}
+```{%- if 'language' in cell.metadata -%}
+{{ cell.metadata.language }}
+{%- elif 'magics_language' in cell.metadata -%}
+{{ cell.metadata.magics_language }}
+{%- elif 'name' in nb.metadata.get('language_info', {}) -%}
+{{ nb.metadata.language_info.name }}
+{%- endif %}
+{{ cell.source }}
+```
+{% endblock input %}
+
+
 {%- block traceback_line -%}
 ```output
 {{ line.rstrip() | strip_ansi }}