WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-09-22 09:35:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

ci: remove markddown-exec from docs pipeline (#3482)

Browse Source 
This PR removes the following changes:
* notebooks that were converted to markdown
* mkdocs.yml file to reference the ipython notebooks rather than the
markdown files
* Makefile install vercel reverted
* hooks for markdown-exec
* notebook conversion jinja2 templates (for converting notebooks to
markdown exec format)
This commit is contained in:
Eugene Yurtsev
2025-02-18 16:16:40 -05:00
committed by GitHub
parent fc834127fd
commit f38784a291
19 changed files with 1638 additions and 1512 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/_scripts/assets/nock_setup_preamble.ts
-75
View File
@@ -1,75 +0,0 @@
import nock, { Definition } from "nock";
import msgpack from "msgpack-lite";
import zlib from "node:zlib";
import fs from "node:fs/promises";
import { Buffer } from "node:buffer";
// deno style imports here because we're running this in the deno jupyter kernel
interface NockCassetteData {
hash: string;
entries: Definition[];
}
// Utility functions for compression & serialization
function compressData(data: NockCassetteData, compressionLevel = 9): string {
const packed = msgpack.encode(data);
const compressed = zlib.deflateSync(packed, { level: compressionLevel });
return compressed.toString("base64");
}
function decompressData(compressedString: string): NockCassetteData {
const decoded = Buffer.from(compressedString, "base64");
const decompressed = zlib.inflateSync(decoded);
return msgpack.decode(decompressed) as NockCassetteData;
}
// deno-lint-ignore no-unused-vars
class HashedCassette {
private recording = true;
constructor(
private readonly cassettePath: string,
private readonly hash: string
) {}
async enter() {
try {
const rawCassette = await fs.readFile(this.cassettePath, "utf-8");
const data = decompressData(rawCassette);
if (data.hash === this.hash) {
this.recording = false;
nock.disableNetConnect();
nock.define(data.entries);
return;
}
} catch (error) {
if (error instanceof Error && error.message.includes("ENOENT")) {
this.recording = true;
} else {
throw error;
}
}
nock.recorder.rec({
dont_print: true,
output_objects: true,
});
}
async exit() {
if (this.recording) {
const entries = nock.recorder.play() as Definition[];
const data = {
hash: this.hash,
entries,
};
const compressed = compressData(data);
await fs.writeFile(this.cassettePath, compressed);
} else {
nock.enableNetConnect();
nock.restore();
nock.cleanAll();
}
}
}
docs/_scripts/assets/vcr_setup_preamble.py
-107
View File
@@ -1,107 +0,0 @@
import base64
import os
import zlib
from types import TracebackType
from typing import Optional, Any, Type
import msgpack
import vcr
os.environ.pop("LANGCHAIN_TRACING_V2", None)
custom_vcr = vcr.VCR()
def compress_data(data: Any, compression_level: int = 9) -> str:
packed = msgpack.packb(data, use_bin_type=True)
compressed = zlib.compress(packed, level=compression_level)
return base64.b64encode(compressed).decode("utf-8")
def decompress_data(compressed_string: str) -> Any:
decoded = base64.b64decode(compressed_string)
decompressed = zlib.decompress(decoded)
return msgpack.unpackb(decompressed, raw=False)
class AdvancedCompressedSerializer:
def serialize(self, cassette_dict: Any) -> str:
return compress_data(cassette_dict)
def deserialize(self, cassette_string: str) -> Any:
return decompress_data(cassette_string)
custom_vcr.register_serializer("advanced_compressed", AdvancedCompressedSerializer())
custom_vcr.serializer = "advanced_compressed"
class HashedCassette:
def __init__(self, cassette_path: str, hash_value: str) -> None:
"""A context manager for using VCR cassettes with an embedded hash value.
Args:
cassette_path (str): The file path of the cassette (independent of hash).
hash_value (str): The expected hash value (e.g. a uuid string).
This class provides a context manager for using VCR cassettes with an embedded hash value.
The hash value is used to ensure that the cassette matches the expected state, and if not,
the cassette is removed or updated with the new hash value.
"""
self.cassette_path: str = cassette_path
self.hash_value: str = hash_value
self.vcr: vcr.VCR = custom_vcr
self.cassette_context: Optional[Any] = None
self.exited: bool = False
def __enter__(self) -> Any:
self.exited: bool = False
# Get the serializer instance from the VCR instance.
serializer = self.vcr.serializers[self.vcr.serializer]
# If the cassette file exists, check its embedded hash.
if os.path.exists(self.cassette_path):
with open(self.cassette_path, "r") as f:
content = f.read()
try:
cassette_data = serializer.deserialize(content)
except Exception as e:
os.remove(self.cassette_path)
else:
existing_hash = cassette_data.get("cassette_hash")
if existing_hash != self.hash_value:
os.remove(self.cassette_path)
# Now enter the VCR cassette context.
self.cassette_context = custom_vcr.use_cassette(
self.cassette_path,
filter_headers=["x-api-key", "authorization"],
record_mode="once",
serializer="advanced_compressed",
)
return self.cassette_context.__enter__()
def __exit__(
self,
exc_type: Optional[Type[BaseException]] = None,
exc_val: Optional[BaseException] = None,
exc_tb: Optional[TracebackType] = None,
) -> Optional[bool]:
if self.exited:
return
self.exited = True
# Exit the VCR cassette context.
result = self.cassette_context.__exit__(exc_type, exc_val, exc_tb)
serializer = self.vcr.serializers[self.vcr.serializer]
# If a cassette was recorded (or updated), open and update its hash.
if os.path.exists(self.cassette_path):
with open(self.cassette_path, "r") as f:
content = f.read()
try:
cassette_data = serializer.deserialize(content)
except Exception as e:
return result
# Update the cassette data with the expected hash.
if cassette_data.get("cassette_hash") != self.hash_value:
cassette_data["cassette_hash"] = self.hash_value
serialized_data = serializer.serialize(cassette_data)
with open(self.cassette_path, "w") as f:
f.write(serialized_data)
return result
docs/_scripts/notebook_convert.py
+2 -109
View File
@@ -1,10 +1,8 @@
import argparse
import ast
import glob
import os
import re
from pathlib import Path
from typing import Literal, Optional
from typing import Literal
import nbformat
from nbconvert.exporters import MarkdownExporter
@@ -352,17 +350,6 @@ exporter = MarkdownExporter(
],
)
md_executable = MarkdownExporter(
preprocessors=[
ExtractAttachmentsPreprocessor,
EscapePreprocessor(markdown_exec_migration=True),
],
template_name="md_executable",
extra_template_basedirs=[
os.path.join(os.path.dirname(__file__), "notebook_convert_templates")
],
)
def convert_notebook(
notebook_path: Path,
@@ -372,99 +359,5 @@ def convert_notebook(
nb = nbformat.read(f, as_version=4)
nb.metadata.mode = mode
if mode == "markdown":
body, _ = exporter.from_notebook_node(nb)
else:
body, _ = md_executable.from_notebook_node(nb)
body, _ = exporter.from_notebook_node(nb)
return body
HERE = Path(__file__).parent
DOCS = HERE.parent / "docs"
# Convert notebooks to markdown
def _convert_notebooks(
*,
output_dir: Optional[Path] = None,
replace: bool = False,
pattern: str = "*.ipynb",
) -> None:
"""Converting notebooks."""
if not output_dir and not replace:
raise ValueError("Either --output_dir or --replace must be specified")
output_dir_path = DOCS if replace else Path(output_dir)
# Get the directory where the script was executed
base_dir = os.getcwd()
# Build the full search pattern using the current working directory as the base
full_pattern = os.path.join(base_dir, args.pattern)
# Use glob with recursive search enabled
matching_files = glob.glob(full_pattern, recursive=True)
paths = [Path(file) for file in matching_files]
file_names = [notebook.name for notebook in paths]
for notebook in paths:
markdown = convert_notebook(notebook, mode="exec")
markdown_path = output_dir_path / notebook.relative_to(DOCS).with_suffix(".md")
markdown_path.parent.mkdir(parents=True, exist_ok=True)
with open(markdown_path, "w") as f:
f.write(markdown)
if replace:
notebook.unlink(missing_ok=False)
if replace:
# The regex will match markdown links that point to *.ipynb files.
# It captures:
# group(1): the link text (inside the square brackets)
# group(2): the file path (without the trailing .ipynb)
link_pattern = r"(?<!!)\[([^\]]+)\]\((?![^)]*//)([^)]+)\.ipynb\)"
def replace_link(match: re.Match) -> str:
link_text = match.group(1)
link_target = match.group(2)
# Reconstruct the file name with the .ipynb extension.
# For example, if link_target is "foo/bar", then linked_file becomes "bar.ipynb".
linked_file = Path(link_target).name + ".ipynb"
# Only update if the notebook was among those converted.
if linked_file in file_names:
# Change the extension from .ipynb to .md
return f"[{link_text}]({link_target}.md)"
# Otherwise, leave the original link intact.
return match.group(0)
# Process all markdown files in the output directory.
for path in output_dir_path.rglob("**/*.md"):
with open(path, "r", encoding="utf-8") as f:
content = f.read()
new_content = re.sub(link_pattern, replace_link, content)
with open(path, "w", encoding="utf-8") as f:
f.write(new_content)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Convert notebooks to markdown")
parser.add_argument(
"--output_dir",
default=None,
help="Directory to output markdown files",
)
parser.add_argument(
"--replace",
action="store_true",
help="Replace original notebooks with markdown files",
)
parser.add_argument(
"--pattern",
default="*.ipynb",
help="Glob pattern to match notebooks to convert",
)
args = parser.parse_args()
_convert_notebooks(
replace=args.replace,
output_dir=args.output_dir,
pattern=args.pattern,
)
docs/_scripts/notebook_convert_templates/md_executable/conf.json
-5
View File
@@ -1,5 +0,0 @@
{
"mimetypes": {
"text/markdown": true
}
}
docs/_scripts/notebook_convert_templates/md_executable/index.md.j2
-38
View File
@@ -1,38 +0,0 @@
{#https://github.com/rdbisme/nbconvert/blob/master/share/jupyter/nbconvert/templates/markdown/index.md.j2#}
{% extends 'markdown/index.md.j2' %}
{% block input %}
```
{%- if 'magics_language' in cell.metadata -%}
{{ cell.metadata.magics_language}}
{%- elif cell.metadata.get('language') == "shell" -%}
shell
{%- elif 'name' in nb.metadata.get('language_info', {}) -%}
{{ nb.metadata.language_info.name }}{% if cell.metadata.exec|default(false) %} exec="on" source="above" session="1"{% if cell.metadata.has_output|default(false) %} result="ansi"{% endif %}{% endif %}
{%- endif %}
{{ cell.source}}
```
{% endblock input %}
{%- block traceback_line -%}
{%- endblock traceback_line -%}
{%- block stream -%}
{%- endblock stream -%}
{%- block data_text scoped -%}
{%- endblock data_text -%}
{%- block data_html scoped -%}
```html
{{ output.data['text/html'] | safe }}
```
{%- endblock data_html -%}
{%- block data_jpg scoped -%}
![](data:image/jpg;base64,{{ output.data['image/jpeg'] }})
{%- endblock data_jpg -%}
{%- block data_png scoped -%}
![](data:image/png;base64,{{ output.data['image/png'] }})
{%- endblock data_png -%}
docs/_scripts/notebook_hooks.py
+1 -118
View File
@@ -2,18 +2,13 @@ import logging
import os
import posixpath
import re
import traceback
from typing import Any, Callable, Dict
from typing import Any, Dict
from markdown import Markdown
from markdown_exec.hooks import SessionHistoryEntry
from mkdocs.structure.files import Files, File
from mkdocs.structure.pages import Page
from pymdownx.superfences import SuperFencesException
from _scripts.generate_api_reference_links import update_markdown_with_imports
from _scripts.notebook_convert import convert_notebook
from _scripts.setup_vcr import load_postamble, load_preamble, _hash_string
logger = logging.getLogger(__name__)
logging.basicConfig()
@@ -163,118 +158,6 @@ def _highlight_code_blocks(markdown: str) -> str:
return markdown
def handle_vcr_setup(
*,
formatter: Callable,
language: str,
code: str,
session: str,
id: str,
md: Markdown,
**kwargs: Dict[str, Any],
) -> Dict[str, Any]:
"""Handle VCR setup in markdown content if necessary."""
try:
if kwargs.get("extra", None) is None:
raise SuperFencesException(
f"error while processing {language} block: extra dict is required"
)
if kwargs["extra"].get("path", None) is None:
raise SuperFencesException(
f"error while processing {language} block: path is required"
)
document_filename = kwargs["extra"]["path"]
if session is None or session == "" and id is None or id == "":
id = _hash_string(code)
if session is not None and session != "":
logger.info(f"new {language} session {session} on page {document_filename}")
cassette_prefix = document_filename.replace(".md", "").replace(os.path.sep, "_")
cassette_dir = os.path.abspath(
os.path.join(os.path.dirname(os.path.dirname(__file__)), "cassettes")
)
os.makedirs(cassette_dir, exist_ok=True)
# Build a unique cassette name.
cassette_name = os.path.join(
cassette_dir,
f"{cassette_prefix}_{session if session else id}_{language}.msgpack.zlib",
)
# Add context manager at start with explicit __enter__ and __exit__ calls
wrapped_lines = [
load_preamble(language, code, cassette_name),
code,
]
if session is None or session == "":
logger.info(
f"no session, adding postamble for {language} in {document_filename}"
)
wrapped_lines.append(load_postamble(language))
transformed_source = "\n".join(wrapped_lines)
# Propagate extras
keep_extras = {
key: value
for key, value in kwargs["extra"].items()
if key
in {
"hl_lines",
}
}
return dict(
transform_source=lambda code: (transformed_source, code),
id=id,
extra=keep_extras,
)
except Exception as e:
raise SuperFencesException(traceback.format_exc()) from e
def handle_vcr_teardown(
*,
formatter: Callable,
language: str,
session: str,
history: list[SessionHistoryEntry],
):
last_inputs = dict(history[-1].inputs)
code = load_postamble(language)
md = last_inputs["md"]
html = False
update_toc = False
document_filename = last_inputs.get("extra", {}).get("path", None)
if document_filename is None:
logger.warning(f"no document filename found while tearing down {session}!")
else:
logger.info(f"tearing down {language} {session} on {document_filename}")
kwargs = dict(
code=code,
session=session,
id=f"{id}_vcr_end",
md=md,
html=html,
update_toc=update_toc,
extra={},
)
# This doesn't actually render anything, we just call the formatter so it
# executes in the same context as the session of which we're disposing.
formatter(**kwargs)
def _on_page_markdown_with_config(
markdown: str,
page: Page,
docs/_scripts/setup_vcr.py
-77
View File
@@ -1,77 +0,0 @@
# A list of patterns that, if found in a code block, will cause us to leave that block unchanged.
import hashlib
import os
from textwrap import dedent
preambles = {
"python": "vcr_setup_preamble.py",
"typescript": "nock_setup_preamble.ts",
}
def _get_python_cassette_init(cassette_name: str, hash_: str) -> str:
return dedent(
f"""
_cassette = HashedCassette('{cassette_name}', '{hash_}')
_cassette.__enter__()
"""
)
def _get_typescript_cassette_init(cassette_name: str, hash_: str) -> str:
return dedent(
f"""
const _cassette = new HashedCassette("{cassette_name}", "{hash_}");
await _cassette.enter();
"""
)
def _get_python_cassette_cleanup() -> str:
return "_cassette.__exit__()"
def _get_typescript_cassette_cleanup() -> str:
return "await _cassette.exit();"
preamble_inits = {
"python": _get_python_cassette_init,
"py": _get_python_cassette_init,
"typescript": _get_typescript_cassette_init,
"ts": _get_typescript_cassette_init,
}
preamble_cleanups = {
"python": _get_python_cassette_cleanup,
"py": _get_python_cassette_cleanup,
"typescript": _get_typescript_cassette_cleanup,
"ts": _get_typescript_cassette_cleanup,
}
def load_preamble(language: str, code: str, cassette_name: str) -> str:
"""Load the source code for the preamble for a given language."""
_assets_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets")
preamble_path = os.path.join(_assets_dir, preambles[language])
with open(preamble_path, "r") as f:
lines = f.readlines()
hash_ = _hash_string(code)
lines.append(preamble_inits[language](cassette_name, hash_))
return "\n".join(lines).strip()
def load_postamble(language: str) -> str:
"""Load the source code for the postamble for a given language."""
return preamble_cleanups[language]()
def _hash_string(input_string: str) -> str:
# Encode the input string to bytes
encoded_string = input_string.encode("utf-8")
# Create a SHA-256 hash object
sha256_hash = hashlib.sha256(encoded_string)
# Get the hexadecimal digest of the hash
return sha256_hash.hexdigest()