mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-08-26 17:42:24 +02:00
docs: add custom hooks for rendering jupyter notebooks (#2067)
This commit is contained in:
Vadym Barda
@@ -24,7 +24,7 @@ export -f execute_notebook
|
||||
|
||||
# Check if custom notebook paths are provided
|
||||
if [ $# -gt 0 ]; then
|
||||
notebooks="$@"
|
||||
notebooks=$(echo "$@" | tr ' ' '\n' | grep -vFf <(echo "$SKIP_NOTEBOOKS"))
|
||||
else
|
||||
# Find all notebooks and filter out those in the skip list
|
||||
notebooks=$(find docs/docs/tutorials docs/docs/how-tos -name "*.ipynb" | grep -v ".ipynb_checkpoints" | grep -vFf <(echo "$SKIP_NOTEBOOKS"))
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
import importlib
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import List, Literal, Optional
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
import nbformat
|
||||
from nbconvert.preprocessors import Preprocessor
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
# Base URL for all class documentation
|
||||
_LANGCHAIN_API_REFERENCE = "https://python.langchain.com/api_reference/"
|
||||
_LANGGRAPH_API_REFERENCE = "https://langchain-ai.github.io/langgraph/reference/"
|
||||
|
||||
|
||||
# (alias/re-exported modules, source module, class, docs namespace)
|
||||
MANUAL_API_REFERENCES_LANGGRAPH = [
|
||||
(
|
||||
["langgraph.prebuilt"],
|
||||
"langgraph.prebuilt.chat_agent_executor",
|
||||
"create_react_agent",
|
||||
"prebuilt",
|
||||
),
|
||||
(["langgraph.prebuilt"], "langgraph.prebuilt.tool_node", "ToolNode", "prebuilt"),
|
||||
(
|
||||
["langgraph.prebuilt"],
|
||||
"langgraph.prebuilt.tool_node",
|
||||
"tools_condition",
|
||||
"prebuilt",
|
||||
),
|
||||
(
|
||||
["langgraph.prebuilt"],
|
||||
"langgraph.prebuilt.tool_node",
|
||||
"InjectedState",
|
||||
"prebuilt",
|
||||
),
|
||||
# Graph
|
||||
(["langgraph.graph"], "langgraph.graph.message", "add_messages", "graphs"),
|
||||
(["langgraph.graph"], "langgraph.graph.state", "StateGraph", "graphs"),
|
||||
(["langgraph.graph"], "langgraph.graph.state", "CompiledStateGraph", "graphs"),
|
||||
([], "langgraph.types", "StreamMode", "types"),
|
||||
(["langgraph.graph"], "langgraph.constants", "START", "constants"),
|
||||
(["langgraph.graph"], "langgraph.constants", "END", "constants"),
|
||||
(["langgraph.constants"], "langgraph.types", "Send", "types"),
|
||||
(["langgraph.constants"], "langgraph.types", "Interrupt", "types"),
|
||||
([], "langgraph.types", "RetryPolicy", "types"),
|
||||
([], "langgraph.checkpoint.base", "Checkpoint", "checkpoints"),
|
||||
([], "langgraph.checkpoint.base", "CheckpointMetadata", "checkpoints"),
|
||||
([], "langgraph.checkpoint.base", "BaseCheckpointSaver", "checkpoints"),
|
||||
([], "langgraph.checkpoint.base", "SerializerProtocol", "checkpoints"),
|
||||
([], "langgraph.checkpoint.serde.jsonplus", "JsonPlusSerializer", "checkpoints"),
|
||||
([], "langgraph.checkpoint.memory", "MemorySaver", "checkpoints"),
|
||||
([], "langgraph.checkpoint.sqlite.aio", "AsyncSqliteSaver", "checkpoints"),
|
||||
([], "langgraph.checkpoint.sqlite", "SqliteSaver", "checkpoints"),
|
||||
([], "langgraph.checkpoint.postgres.aio", "AsyncPostgresSaver", "checkpoints"),
|
||||
([], "langgraph.checkpoint.postgres", "PostgresSaver", "checkpoints"),
|
||||
]
|
||||
|
||||
WELL_KNOWN_LANGGRAPH_OBJECTS = {
|
||||
(module_, class_): (source_module, namespace)
|
||||
for (modules, source_module, class_, namespace) in MANUAL_API_REFERENCES_LANGGRAPH
|
||||
for module_ in modules + [source_module]
|
||||
}
|
||||
|
||||
|
||||
def _make_regular_expression(pkg_prefix: str) -> re.Pattern:
|
||||
if not pkg_prefix.isidentifier():
|
||||
raise ValueError(f"Invalid package prefix: {pkg_prefix}")
|
||||
return re.compile(
|
||||
r"from\s+(" + pkg_prefix + "(?:_\w+)?(?:\.\w+)*?)\s+import\s+"
|
||||
r"((?:\w+(?:,\s*)?)*" # Match zero or more words separated by a comma+optional ws
|
||||
r"(?:\s*\(.*?\))?)", # Match optional parentheses block
|
||||
re.DOTALL, # Match newlines as well
|
||||
)
|
||||
|
||||
|
||||
# Regular expression to match langchain import lines
|
||||
_IMPORT_LANGCHAIN_RE = _make_regular_expression("langchain")
|
||||
_IMPORT_LANGGRAPH_RE = _make_regular_expression("langgraph")
|
||||
|
||||
|
||||
def _get_full_module_name(module_path, class_name) -> Optional[str]:
|
||||
"""Get full module name using inspect"""
|
||||
try:
|
||||
module = importlib.import_module(module_path)
|
||||
class_ = getattr(module, class_name)
|
||||
module = inspect.getmodule(class_)
|
||||
if module is None:
|
||||
# For constants, inspect.getmodule() might return None
|
||||
# In this case, we'll return the original module_path
|
||||
return module_path
|
||||
return module.__name__
|
||||
except AttributeError as e:
|
||||
logger.warning(f"Could not find module for {class_name}, {e}")
|
||||
return None
|
||||
except ImportError as e:
|
||||
logger.warning(f"Failed to load for class {class_name}, {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _get_doc_title(data: str, file_name: str) -> str:
|
||||
try:
|
||||
return re.findall(r"^#\s*(.*)", data, re.MULTILINE)[0]
|
||||
except IndexError:
|
||||
pass
|
||||
# Parse the rst-style titles
|
||||
try:
|
||||
return re.findall(r"^(.*)\n=+\n", data, re.MULTILINE)[0]
|
||||
except IndexError:
|
||||
return file_name
|
||||
|
||||
|
||||
class ImportInformation(TypedDict):
|
||||
imported: str # imported class name
|
||||
source: str # module path
|
||||
docs: str # URL to the documentation
|
||||
title: str # Title of the document
|
||||
|
||||
|
||||
def _get_imports(
|
||||
code: str, doc_title: str, package_ecosystem: Literal["langchain", "langgraph"]
|
||||
) -> List[ImportInformation]:
|
||||
"""Get imports from the given code block.
|
||||
|
||||
Args:
|
||||
code: Python code block from which to extract imports
|
||||
doc_title: Title of the document
|
||||
package_ecosystem: "langchain" or "langgraph". The two live in different
|
||||
repositories and have separate documentation sites.
|
||||
|
||||
Returns:
|
||||
List of import information for the given code block
|
||||
"""
|
||||
imports = []
|
||||
|
||||
if package_ecosystem == "langchain":
|
||||
pattern = _IMPORT_LANGCHAIN_RE
|
||||
elif package_ecosystem == "langgraph":
|
||||
pattern = _IMPORT_LANGGRAPH_RE
|
||||
else:
|
||||
raise ValueError(f"Invalid package ecosystem: {package_ecosystem}")
|
||||
|
||||
for import_match in pattern.finditer(code):
|
||||
module = import_match.group(1)
|
||||
if "pydantic_v1" in module:
|
||||
continue
|
||||
imports_str = (
|
||||
import_match.group(2).replace("(\n", "").replace("\n)", "")
|
||||
) # Handle newlines within parentheses
|
||||
# remove any newline and spaces, then split by comma
|
||||
imported_classes = [
|
||||
imp.strip()
|
||||
for imp in re.split(r",\s*", imports_str.replace("\n", ""))
|
||||
if imp.strip()
|
||||
]
|
||||
for class_name in imported_classes:
|
||||
module_path = _get_full_module_name(module, class_name)
|
||||
if not module_path:
|
||||
continue
|
||||
if len(module_path.split(".")) < 2:
|
||||
continue
|
||||
|
||||
if package_ecosystem == "langchain":
|
||||
pkg = module_path.split(".")[0].replace("langchain_", "")
|
||||
top_level_mod = module_path.split(".")[1]
|
||||
|
||||
url = (
|
||||
_LANGCHAIN_API_REFERENCE
|
||||
+ pkg
|
||||
+ "/"
|
||||
+ top_level_mod
|
||||
+ "/"
|
||||
+ module_path
|
||||
+ "."
|
||||
+ class_name
|
||||
+ ".html"
|
||||
)
|
||||
elif package_ecosystem == "langgraph":
|
||||
if (module, class_name) not in WELL_KNOWN_LANGGRAPH_OBJECTS:
|
||||
# Likely not documented yet
|
||||
continue
|
||||
|
||||
source_module, namespace = WELL_KNOWN_LANGGRAPH_OBJECTS[
|
||||
(module, class_name)
|
||||
]
|
||||
url = (
|
||||
_LANGGRAPH_API_REFERENCE
|
||||
+ namespace
|
||||
+ "/#"
|
||||
+ source_module
|
||||
+ "."
|
||||
+ class_name
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid package ecosystem: {package_ecosystem}")
|
||||
|
||||
# Add the import information to our list
|
||||
imports.append(
|
||||
{
|
||||
"imported": class_name,
|
||||
"source": module,
|
||||
"docs": url,
|
||||
"title": doc_title,
|
||||
}
|
||||
)
|
||||
|
||||
return imports
|
||||
|
||||
|
||||
class ImportPreprocessor(Preprocessor):
|
||||
"""A preprocessor to replace imports in each Python code cell with links to their
|
||||
documentation and append the import info in a comment."""
|
||||
|
||||
def preprocess(self, nb, resources):
|
||||
self.all_imports = []
|
||||
file_name = os.path.basename(resources.get("metadata", {}).get("name", ""))
|
||||
_DOC_TITLE = _get_doc_title(nb.cells[0].source, file_name)
|
||||
|
||||
cells = []
|
||||
for cell in nb.cells:
|
||||
if cell.cell_type == "code":
|
||||
cells.append(cell)
|
||||
imports = _get_imports(
|
||||
cell.source, _DOC_TITLE, "langchain"
|
||||
) + _get_imports(cell.source, _DOC_TITLE, "langgraph")
|
||||
if not imports:
|
||||
continue
|
||||
|
||||
cells.append(
|
||||
nbformat.v4.new_markdown_cell(
|
||||
source=f"""
|
||||
<div>
|
||||
<b>API Reference:</b>
|
||||
{' | '.join(f'<a href="{imp["docs"]}">{imp["imported"]}</a>' for imp in imports)}
|
||||
</div>
|
||||
"""
|
||||
)
|
||||
)
|
||||
else:
|
||||
cells.append(cell)
|
||||
nb.cells = cells
|
||||
return nb, resources
|
||||
@@ -0,0 +1,126 @@
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import nbformat
|
||||
from nbconvert.exporters import MarkdownExporter
|
||||
from nbconvert.preprocessors import Preprocessor
|
||||
|
||||
from generate_api_reference_links import ImportPreprocessor
|
||||
|
||||
|
||||
class EscapePreprocessor(Preprocessor):
|
||||
def preprocess_cell(self, cell, resources, cell_index):
|
||||
if cell.cell_type == "markdown":
|
||||
# rewrite markdown links to html links (excluding image links)
|
||||
cell.source = re.sub(
|
||||
r"(?<!!)\[([^\]]*)\]\((?![^\)]*//)([^)]*)(?:\.ipynb)?\)",
|
||||
r'<a href="\2">\1</a>',
|
||||
cell.source,
|
||||
)
|
||||
# Fix image paths in <img> tags
|
||||
cell.source = re.sub(
|
||||
r'<img\s+src="\.?/img/([^"]+)"', r'<img src="../img/\1"', cell.source
|
||||
)
|
||||
|
||||
elif cell.cell_type == "code":
|
||||
# escape ``` in code
|
||||
cell.source = cell.source.replace("```", r"\`\`\`")
|
||||
# escape ``` in output
|
||||
if "outputs" in cell:
|
||||
filter_out = set()
|
||||
for i, output in enumerate(cell["outputs"]):
|
||||
if "text" in output:
|
||||
if not output["text"].strip():
|
||||
filter_out.add(i)
|
||||
continue
|
||||
|
||||
value = output["text"].replace("```", r"\`\`\`")
|
||||
# handle a funky case w/ references in text
|
||||
value = re.sub(r"\[(\d+)\](?=\[(\d+)\])", r"[\1]\\", value)
|
||||
output["text"] = value
|
||||
elif "data" in output:
|
||||
for key, value in output["data"].items():
|
||||
if isinstance(value, str):
|
||||
value = value.replace("```", r"\`\`\`")
|
||||
# handle a funky case w/ references in text
|
||||
output["data"][key] = re.sub(
|
||||
r"\[(\d+)\](?=\[(\d+)\])", r"[\1]\\", value
|
||||
)
|
||||
cell["outputs"] = [
|
||||
output
|
||||
for i, output in enumerate(cell["outputs"])
|
||||
if i not in filter_out
|
||||
]
|
||||
|
||||
return cell, resources
|
||||
|
||||
|
||||
class ExtractAttachmentsPreprocessor(Preprocessor):
|
||||
"""
|
||||
Extracts all of the outputs from the notebook file. The extracted
|
||||
outputs are returned in the 'resources' dictionary.
|
||||
"""
|
||||
|
||||
def preprocess_cell(self, cell, resources, cell_index):
|
||||
"""
|
||||
Apply a transformation on each cell,
|
||||
Parameters
|
||||
----------
|
||||
cell : NotebookNode cell
|
||||
Notebook cell being processed
|
||||
resources : dictionary
|
||||
Additional resources used in the conversion process. Allows
|
||||
preprocessors to pass variables into the Jinja engine.
|
||||
cell_index : int
|
||||
Index of the cell being processed (see base.py)
|
||||
"""
|
||||
|
||||
# Get files directory if it has been specified
|
||||
|
||||
# Make sure outputs key exists
|
||||
if not isinstance(resources["outputs"], dict):
|
||||
resources["outputs"] = {}
|
||||
|
||||
# Loop through all of the attachments in the cell
|
||||
for name, attach in cell.get("attachments", {}).items():
|
||||
for mime, data in attach.items():
|
||||
if mime not in {
|
||||
"image/png",
|
||||
"image/jpeg",
|
||||
"image/svg+xml",
|
||||
"application/pdf",
|
||||
}:
|
||||
continue
|
||||
|
||||
# attachments are pre-rendered. Only replace markdown-formatted
|
||||
# images with the following logic
|
||||
attach_str = f"({name})"
|
||||
if attach_str in cell.source:
|
||||
data = f"(data:{mime};base64,{data})"
|
||||
cell.source = cell.source.replace(attach_str, data)
|
||||
|
||||
return cell, resources
|
||||
|
||||
|
||||
exporter = MarkdownExporter(
|
||||
preprocessors=[
|
||||
EscapePreprocessor,
|
||||
ExtractAttachmentsPreprocessor,
|
||||
ImportPreprocessor,
|
||||
],
|
||||
template_name="mdoutput",
|
||||
extra_template_basedirs=[
|
||||
os.path.join(os.path.dirname(__file__), "notebook_convert_templates")
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def convert_notebook(
|
||||
notebook_path: Path,
|
||||
) -> Path:
|
||||
with open(notebook_path) as f:
|
||||
nb = nbformat.read(f, as_version=4)
|
||||
|
||||
body, _ = exporter.from_notebook_node(nb)
|
||||
return body
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"mimetypes": {
|
||||
"text/markdown": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{% extends 'markdown/index.md.j2' %}
|
||||
|
||||
{%- block traceback_line -%}
|
||||
```output
|
||||
{{ line.rstrip() | strip_ansi }}
|
||||
```
|
||||
{%- endblock traceback_line -%}
|
||||
|
||||
{%- block stream -%}
|
||||
```output
|
||||
{{ output.text.rstrip() }}
|
||||
```
|
||||
{%- endblock stream -%}
|
||||
|
||||
{%- block data_text scoped -%}
|
||||
```output
|
||||
{{ output.data['text/plain'].rstrip() }}
|
||||
```
|
||||
{%- endblock data_text -%}
|
||||
|
||||
{%- block data_html scoped -%}
|
||||
```html
|
||||
{{ output.data['text/html'] | safe }}
|
||||
```
|
||||
{%- endblock data_html -%}
|
||||
|
||||
{%- block data_jpg scoped -%}
|
||||
|
||||
{%- endblock data_jpg -%}
|
||||
|
||||
{%- block data_png scoped -%}
|
||||
|
||||
{%- endblock data_png -%}
|
||||
@@ -0,0 +1,40 @@
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
from mkdocs.structure.pages import Page
|
||||
from mkdocs.structure.files import Files, File
|
||||
from notebook_convert import convert_notebook
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig()
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
|
||||
class NotebookFile(File):
|
||||
def is_documentation_page(self):
|
||||
return True
|
||||
|
||||
|
||||
def on_files(files: Files, **kwargs: Dict[str, Any]):
|
||||
new_files = Files([])
|
||||
for file in files:
|
||||
if file.src_path.endswith(".ipynb"):
|
||||
new_file = NotebookFile(
|
||||
path=file.src_path,
|
||||
src_dir=file.src_dir,
|
||||
dest_dir=file.dest_dir,
|
||||
use_directory_urls=file.use_directory_urls,
|
||||
)
|
||||
new_files.append(new_file)
|
||||
else:
|
||||
new_files.append(file)
|
||||
return new_files
|
||||
|
||||
|
||||
def on_page_markdown(markdown: str, page: Page, **kwargs: Dict[str, Any]):
|
||||
if page.file.src_path.endswith(".ipynb"):
|
||||
logger.info("Processing Jupyter notebook: %s", page.file.src_path)
|
||||
body = convert_notebook(page.file.abs_src_path)
|
||||
return body
|
||||
|
||||
return markdown
|
||||
Reference in New Issue
Block a user