WEBDIENSTE/langgraph
1
0
Fork 0
mirror of https://github.com/langchain-ai/langgraph.git synced 2026-09-09 11:17:53 +02:00
Code Issues Packages Projects Releases Wiki Activity

docs: get ts execution working in build pipeline

Browse Source
This commit is contained in:
Ben Burns
2025-02-13 21:57:37 -08:00
parent f2faa39ca9
commit 441923282c
10 changed files with 5835 additions and 485 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/_scripts/check_secrets.py
+56
View File
@@ -0,0 +1,56 @@
import base64
import json
import os
import re
import sys
import zlib
from typing import Any
import msgpack
def compress_data(data: Any, compression_level: int = 9) -> str:
packed = msgpack.packb(data, use_bin_type=True)
compressed = zlib.compress(packed, level=compression_level)
return base64.b64encode(compressed).decode("utf-8")
def decompress_data(compressed_string: str) -> Any:
decoded = base64.b64decode(compressed_string)
decompressed = zlib.decompress(decoded)
return msgpack.unpackb(decompressed, raw=False)
if __name__ == "__main__":
for file in os.listdir("cassettes"):
if file.endswith(".msgpack.zlib"):
with open(
f"cassettes/{file}",
"r",
) as f:
data = f.read()
decompressed = decompress_data(data)
decompressed = json.dumps(decompressed, default=str)
if "x-api-key" in decompressed.lower():
print(f"Found secret (x-api-key) in {file}!", file=sys.stderr)
print(decompressed, file=sys.stderr)
if "Bearer: " in decompressed.lower():
print(f"Found potential secret (bearer token) in {file}!", file=sys.stderr)
print(decompressed, file=sys.stderr)
if match := re.match(r"sk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}", decompressed.lower()):
print(f"OpenAI user API key found in {file}!", file=sys.stderr)
print(match.group(0), file=sys.stderr)
if match := re.match(r"sk-proj-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}", decompressed.lower()):
print(f"OpenAI user project API key found in {file}!", file=sys.stderr)
print(match.group(0), file=sys.stderr)
if match := re.match(r"sk-proj-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}", decompressed.lower()):
print(f"OpenAI user project API key found in {file}!", file=sys.stderr)
print(match.group(0), file=sys.stderr)
for service_id in re.compile(r"^[A-Za-z0-9]+(-*[A-Za-z0-9]+)*$").finditer(decompressed):
if match := re.match(f"sk-{service_id.group(0)}-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}", decompressed.lower()):
print(f"OpenAI service key found in {file}!", file=sys.stderr)
print(match.group(0), file=sys.stderr)
if match := re.match(r"sk-ant-[A-Za-z0-9_-]{101}", decompressed.lower()):
print(f"Anthropic API key found in {file}!", file=sys.stderr)
print(match.group(0), file=sys.stderr)
docs/_scripts/hook_state.py
+3
View File
@@ -0,0 +1,3 @@
hook_state = {
"document_filename": "__UNKNOWN__",
}
docs/_scripts/notebook_hooks.py
+21 -46
View File
@@ -6,11 +6,14 @@ import traceback
from typing import Any, Callable, Dict
from markdown import Markdown
from markdown_exec.hooks import SessionHistoryEntry
from mkdocs.structure.files import Files, File
from mkdocs.structure.pages import Page
from pymdownx.superfences import SuperFencesException
from _scripts.hook_state import hook_state
from markdown_exec.hooks import SessionHistoryEntry
from _scripts.generate_api_reference_links import update_markdown_with_imports
from _scripts.notebook_convert import convert_notebook
from _scripts.setup_vcr import load_postamble, load_preamble, _hash_string
@@ -62,29 +65,6 @@ def on_files(files: Files, **kwargs: Dict[str, Any]):
return new_files
def _add_path_to_code_blocks(markdown: str, page: Page) -> str:
"""Add the path to the code blocks."""
code_block_pattern = re.compile(
r"(?P<indent>[ \t]*)```(?P<language>\w+)[ ]*(?P<attributes>[^\n]*)\n"
r"(?P<code>((?:.*\n)*?))" # Capture the code inside the block using named group
r"(?P=indent)```" # Match closing backticks with the same indentation
)
def replace_code_block_header(match: re.Match) -> str:
indent = match.group("indent")
language = match.group("language")
attributes = match.group("attributes").rstrip()
if 'exec="on"' not in attributes:
# Return original code block
return match.group(0)
code = match.group("code")
return f'{indent}```{language} {attributes} path="{page.file.src_path}"\n{code}{indent}```'
return code_block_pattern.sub(replace_code_block_header, markdown)
def _highlight_code_blocks(markdown: str) -> str:
"""Find code blocks with highlight comments and add hl_lines attribute.
@@ -162,7 +142,6 @@ def _highlight_code_blocks(markdown: str) -> str:
markdown = code_block_pattern.sub(replace_highlight_comments, markdown)
return markdown
def handle_vcr_setup(
*,
formatter: Callable,
@@ -174,26 +153,20 @@ def handle_vcr_setup(
**kwargs: Dict[str, Any],
) -> Dict[str, Any]:
"""Handle VCR setup in markdown content if necessary."""
logger.info(f"handle_vcr_setup: {hook_state['document_filename']}")
try:
if kwargs.get("extra", None) is None:
if hook_state['document_filename'] == '__UNKNOWN__':
raise SuperFencesException(
f"error while processing {language} block: extra dict is required"
f"error while processing {language} block: document filename is unknown"
)
if kwargs["extra"].get("path", None) is None:
raise SuperFencesException(
f"error while processing {language} block: path is required"
)
document_filename = kwargs["extra"]["path"]
if session is None or session == "" and id is None or id == "":
id = _hash_string(code)
if session is not None and session != "":
logger.info(f"new {language} session {session} on page {document_filename}")
logger.info(f"new {language} session {session} on page {hook_state['document_filename']}")
cassette_prefix = document_filename.replace(".md", "").replace(os.path.sep, "_")
cassette_prefix = hook_state['document_filename'].replace(".md", "").replace(os.path.sep, "_")
cassette_dir = os.path.abspath(
os.path.join(os.path.dirname(os.path.dirname(__file__)), "cassettes")
@@ -215,7 +188,7 @@ def handle_vcr_setup(
if session is None or session == "":
logger.info(
f"no session, adding postamble for {language} in {document_filename}"
f"no session, adding postamble for {language} in {hook_state['document_filename']}"
)
wrapped_lines.append(load_postamble(language))
@@ -234,7 +207,7 @@ def handle_vcr_setup(
return dict(
transform_source=lambda code: (transformed_source, code),
id=id,
extra=keep_extras,
extra={ **keep_extras, "path": hook_state['document_filename'] },
)
except Exception as e:
raise SuperFencesException(traceback.format_exc()) from e
@@ -253,12 +226,12 @@ def handle_vcr_teardown(
html = False
update_toc = False
document_filename = last_inputs.get("extra", {}).get("path", None)
path = last_inputs.get("extra", {}).get("path", None)
if document_filename is None:
if path is None:
logger.warning(f"no document filename found while tearing down {session}!")
else:
logger.info(f"tearing down {language} {session} on {document_filename}")
logger.info(f"tearing down {language} {session} on {path}")
kwargs = dict(
code=code,
@@ -296,11 +269,6 @@ def _on_page_markdown_with_config(
# Apply highlight comments to code blocks
markdown = _highlight_code_blocks(markdown)
# Add file path as an attribute to code blocks that are executable.
# This file path is used to associate fixtures with the executable code
# which can be used in CI to test the docs without making network requests.
markdown = _add_path_to_code_blocks(markdown, page)
if remove_base64_images:
# Remove base64 encoded images from markdown
markdown = re.sub(r"!\[.*?\]\(data:image/+;base64,[^\)]+\)", "", markdown)
@@ -309,6 +277,8 @@ def _on_page_markdown_with_config(
def on_page_markdown(markdown: str, page: Page, **kwargs: Dict[str, Any]):
logger.info(f"on_page_markdown: {page.file.src_path}")
hook_state['document_filename'] = page.file.src_path
return _on_page_markdown_with_config(
markdown,
page,
@@ -370,3 +340,8 @@ def on_post_build(config):
+ suffix
)
write_html(config["site_dir"], old_html_path, new_html_path)
def on_pre_page(page: Page, **kwargs: Dict[str, Any]):
logger.info(f"on_pre_page: {page.file.src_path}")
hook_state['document_filename'] = page.file.src_path
return page