mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-09-10 03:37:51 +02:00
docs: get ts execution working in build pipeline
This commit is contained in:
Ben Burns
+873
File diff suppressed because one or more lines are too long
@@ -0,0 +1,3 @@
|
||||
nodeLinker: node-modules
|
||||
|
||||
yarnPath: .yarn/releases/yarn-3.5.1.cjs
|
||||
+12
-3
@@ -13,7 +13,17 @@ build-prebuilt:
|
||||
poetry run python -m _scripts.third_party_page.get_download_stats stats.yml
|
||||
poetry run python -m _scripts.third_party_page.create_third_party_page stats.yml docs/prebuilt.md --language python
|
||||
|
||||
build-docs: build-typedoc build-prebuilt
|
||||
grab-langgraphjs:
|
||||
if [ ! -d "langgraphjs" ]; then \
|
||||
git clone https://github.com/langchain-ai/langgraphjs.git; \
|
||||
else \
|
||||
cd langgraphjs && git checkout main && git pull; \
|
||||
fi
|
||||
cd langgraphjs && yarn
|
||||
cd langgraphjs && yarn build
|
||||
yarn
|
||||
|
||||
build-docs: build-typedoc build-prebuilt grab-langgraphjs
|
||||
poetry run python -m mkdocs build --clean -f mkdocs.yml --strict
|
||||
|
||||
llms-text:
|
||||
@@ -32,7 +42,6 @@ install-vercel-deps:
|
||||
poetry run tslab install --python=python3
|
||||
poetry run jupyter kernelspec list
|
||||
|
||||
|
||||
tests:
|
||||
# RUn unit tests
|
||||
poetry run pytest tests/unit_tests
|
||||
@@ -45,7 +54,7 @@ vercel-build-docs: install-vercel-deps
|
||||
serve-clean-docs: clean-docs
|
||||
poetry run python -m mkdocs serve -c -f mkdocs.yml --strict -w ../libs/langgraph
|
||||
|
||||
serve-docs: build-typedoc
|
||||
serve-docs: build-typedoc grab-langgraphjs
|
||||
poetry run python -m mkdocs serve -f mkdocs.yml -w ../libs/langgraph -w ../libs/checkpoint -w ../libs/sdk-py --dirty
|
||||
|
||||
clean-docs:
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import zlib
|
||||
from typing import Any
|
||||
|
||||
import msgpack
|
||||
|
||||
|
||||
def compress_data(data: Any, compression_level: int = 9) -> str:
|
||||
packed = msgpack.packb(data, use_bin_type=True)
|
||||
compressed = zlib.compress(packed, level=compression_level)
|
||||
return base64.b64encode(compressed).decode("utf-8")
|
||||
|
||||
|
||||
def decompress_data(compressed_string: str) -> Any:
|
||||
decoded = base64.b64decode(compressed_string)
|
||||
decompressed = zlib.decompress(decoded)
|
||||
return msgpack.unpackb(decompressed, raw=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for file in os.listdir("cassettes"):
|
||||
if file.endswith(".msgpack.zlib"):
|
||||
with open(
|
||||
f"cassettes/{file}",
|
||||
"r",
|
||||
) as f:
|
||||
data = f.read()
|
||||
|
||||
decompressed = decompress_data(data)
|
||||
decompressed = json.dumps(decompressed, default=str)
|
||||
if "x-api-key" in decompressed.lower():
|
||||
print(f"Found secret (x-api-key) in {file}!", file=sys.stderr)
|
||||
print(decompressed, file=sys.stderr)
|
||||
if "Bearer: " in decompressed.lower():
|
||||
print(f"Found potential secret (bearer token) in {file}!", file=sys.stderr)
|
||||
print(decompressed, file=sys.stderr)
|
||||
if match := re.match(r"sk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}", decompressed.lower()):
|
||||
print(f"OpenAI user API key found in {file}!", file=sys.stderr)
|
||||
print(match.group(0), file=sys.stderr)
|
||||
if match := re.match(r"sk-proj-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}", decompressed.lower()):
|
||||
print(f"OpenAI user project API key found in {file}!", file=sys.stderr)
|
||||
print(match.group(0), file=sys.stderr)
|
||||
if match := re.match(r"sk-proj-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}", decompressed.lower()):
|
||||
print(f"OpenAI user project API key found in {file}!", file=sys.stderr)
|
||||
print(match.group(0), file=sys.stderr)
|
||||
for service_id in re.compile(r"^[A-Za-z0-9]+(-*[A-Za-z0-9]+)*$").finditer(decompressed):
|
||||
if match := re.match(f"sk-{service_id.group(0)}-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}", decompressed.lower()):
|
||||
print(f"OpenAI service key found in {file}!", file=sys.stderr)
|
||||
print(match.group(0), file=sys.stderr)
|
||||
if match := re.match(r"sk-ant-[A-Za-z0-9_-]{101}", decompressed.lower()):
|
||||
print(f"Anthropic API key found in {file}!", file=sys.stderr)
|
||||
print(match.group(0), file=sys.stderr)
|
||||
@@ -0,0 +1,3 @@
|
||||
hook_state = {
|
||||
"document_filename": "__UNKNOWN__",
|
||||
}
|
||||
@@ -6,11 +6,14 @@ import traceback
|
||||
from typing import Any, Callable, Dict
|
||||
|
||||
from markdown import Markdown
|
||||
from markdown_exec.hooks import SessionHistoryEntry
|
||||
from mkdocs.structure.files import Files, File
|
||||
from mkdocs.structure.pages import Page
|
||||
from pymdownx.superfences import SuperFencesException
|
||||
|
||||
from _scripts.hook_state import hook_state
|
||||
from markdown_exec.hooks import SessionHistoryEntry
|
||||
|
||||
|
||||
from _scripts.generate_api_reference_links import update_markdown_with_imports
|
||||
from _scripts.notebook_convert import convert_notebook
|
||||
from _scripts.setup_vcr import load_postamble, load_preamble, _hash_string
|
||||
@@ -62,29 +65,6 @@ def on_files(files: Files, **kwargs: Dict[str, Any]):
|
||||
return new_files
|
||||
|
||||
|
||||
def _add_path_to_code_blocks(markdown: str, page: Page) -> str:
|
||||
"""Add the path to the code blocks."""
|
||||
code_block_pattern = re.compile(
|
||||
r"(?P<indent>[ \t]*)```(?P<language>\w+)[ ]*(?P<attributes>[^\n]*)\n"
|
||||
r"(?P<code>((?:.*\n)*?))" # Capture the code inside the block using named group
|
||||
r"(?P=indent)```" # Match closing backticks with the same indentation
|
||||
)
|
||||
|
||||
def replace_code_block_header(match: re.Match) -> str:
|
||||
indent = match.group("indent")
|
||||
language = match.group("language")
|
||||
attributes = match.group("attributes").rstrip()
|
||||
|
||||
if 'exec="on"' not in attributes:
|
||||
# Return original code block
|
||||
return match.group(0)
|
||||
|
||||
code = match.group("code")
|
||||
return f'{indent}```{language} {attributes} path="{page.file.src_path}"\n{code}{indent}```'
|
||||
|
||||
return code_block_pattern.sub(replace_code_block_header, markdown)
|
||||
|
||||
|
||||
def _highlight_code_blocks(markdown: str) -> str:
|
||||
"""Find code blocks with highlight comments and add hl_lines attribute.
|
||||
|
||||
@@ -162,7 +142,6 @@ def _highlight_code_blocks(markdown: str) -> str:
|
||||
markdown = code_block_pattern.sub(replace_highlight_comments, markdown)
|
||||
return markdown
|
||||
|
||||
|
||||
def handle_vcr_setup(
|
||||
*,
|
||||
formatter: Callable,
|
||||
@@ -174,26 +153,20 @@ def handle_vcr_setup(
|
||||
**kwargs: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
"""Handle VCR setup in markdown content if necessary."""
|
||||
logger.info(f"handle_vcr_setup: {hook_state['document_filename']}")
|
||||
try:
|
||||
if kwargs.get("extra", None) is None:
|
||||
if hook_state['document_filename'] == '__UNKNOWN__':
|
||||
raise SuperFencesException(
|
||||
f"error while processing {language} block: extra dict is required"
|
||||
f"error while processing {language} block: document filename is unknown"
|
||||
)
|
||||
|
||||
if kwargs["extra"].get("path", None) is None:
|
||||
raise SuperFencesException(
|
||||
f"error while processing {language} block: path is required"
|
||||
)
|
||||
|
||||
document_filename = kwargs["extra"]["path"]
|
||||
|
||||
if session is None or session == "" and id is None or id == "":
|
||||
id = _hash_string(code)
|
||||
|
||||
if session is not None and session != "":
|
||||
logger.info(f"new {language} session {session} on page {document_filename}")
|
||||
logger.info(f"new {language} session {session} on page {hook_state['document_filename']}")
|
||||
|
||||
cassette_prefix = document_filename.replace(".md", "").replace(os.path.sep, "_")
|
||||
cassette_prefix = hook_state['document_filename'].replace(".md", "").replace(os.path.sep, "_")
|
||||
|
||||
cassette_dir = os.path.abspath(
|
||||
os.path.join(os.path.dirname(os.path.dirname(__file__)), "cassettes")
|
||||
@@ -215,7 +188,7 @@ def handle_vcr_setup(
|
||||
|
||||
if session is None or session == "":
|
||||
logger.info(
|
||||
f"no session, adding postamble for {language} in {document_filename}"
|
||||
f"no session, adding postamble for {language} in {hook_state['document_filename']}"
|
||||
)
|
||||
wrapped_lines.append(load_postamble(language))
|
||||
|
||||
@@ -234,7 +207,7 @@ def handle_vcr_setup(
|
||||
return dict(
|
||||
transform_source=lambda code: (transformed_source, code),
|
||||
id=id,
|
||||
extra=keep_extras,
|
||||
extra={ **keep_extras, "path": hook_state['document_filename'] },
|
||||
)
|
||||
except Exception as e:
|
||||
raise SuperFencesException(traceback.format_exc()) from e
|
||||
@@ -253,12 +226,12 @@ def handle_vcr_teardown(
|
||||
html = False
|
||||
update_toc = False
|
||||
|
||||
document_filename = last_inputs.get("extra", {}).get("path", None)
|
||||
path = last_inputs.get("extra", {}).get("path", None)
|
||||
|
||||
if document_filename is None:
|
||||
if path is None:
|
||||
logger.warning(f"no document filename found while tearing down {session}!")
|
||||
else:
|
||||
logger.info(f"tearing down {language} {session} on {document_filename}")
|
||||
logger.info(f"tearing down {language} {session} on {path}")
|
||||
|
||||
kwargs = dict(
|
||||
code=code,
|
||||
@@ -296,11 +269,6 @@ def _on_page_markdown_with_config(
|
||||
# Apply highlight comments to code blocks
|
||||
markdown = _highlight_code_blocks(markdown)
|
||||
|
||||
# Add file path as an attribute to code blocks that are executable.
|
||||
# This file path is used to associate fixtures with the executable code
|
||||
# which can be used in CI to test the docs without making network requests.
|
||||
markdown = _add_path_to_code_blocks(markdown, page)
|
||||
|
||||
if remove_base64_images:
|
||||
# Remove base64 encoded images from markdown
|
||||
markdown = re.sub(r"!\[.*?\]\(data:image/+;base64,[^\)]+\)", "", markdown)
|
||||
@@ -309,6 +277,8 @@ def _on_page_markdown_with_config(
|
||||
|
||||
|
||||
def on_page_markdown(markdown: str, page: Page, **kwargs: Dict[str, Any]):
|
||||
logger.info(f"on_page_markdown: {page.file.src_path}")
|
||||
hook_state['document_filename'] = page.file.src_path
|
||||
return _on_page_markdown_with_config(
|
||||
markdown,
|
||||
page,
|
||||
@@ -370,3 +340,8 @@ def on_post_build(config):
|
||||
+ suffix
|
||||
)
|
||||
write_html(config["site_dir"], old_html_path, new_html_path)
|
||||
|
||||
def on_pre_page(page: Page, **kwargs: Dict[str, Any]):
|
||||
logger.info(f"on_pre_page: {page.file.src_path}")
|
||||
hook_state['document_filename'] = page.file.src_path
|
||||
return page
|
||||
|
||||
@@ -11,10 +11,14 @@ We will use [messages](../concepts/low_level.md/#messagesstate) in our examples.
|
||||
|
||||
First, let's install langgraph:
|
||||
|
||||
```python
|
||||
%%capture --no-stderr
|
||||
%pip install -U langgraph
|
||||
```
|
||||
=== "Python"
|
||||
```shell
|
||||
pip install -U langgraph
|
||||
```
|
||||
=== "TypeScript"
|
||||
```shell
|
||||
npm install @langchain/langgraph
|
||||
```
|
||||
|
||||
<div class="admonition tip">
|
||||
<p class="admonition-title">Set up <a href="https://smith.langchain.com">LangSmith</a> for better debugging</p>
|
||||
@@ -23,25 +27,37 @@ First, let's install langgraph:
|
||||
</p>
|
||||
</div>
|
||||
|
||||
|
||||
## Example graph
|
||||
|
||||
### Define state
|
||||
|
||||
[State](../concepts/low_level.md/#state) in LangGraph can be a `TypedDict`, `Pydantic` model, or dataclass. Below we will use `TypedDict`. See [this guide](../how-tos/state-model.ipynb) for detail on using Pydantic.
|
||||
|
||||
By default, graphs will have the same input and output schema, and the state determines that schema. See [this guide](../how-tos/input_output_schema.ipynb) for how to define distinct input and output schemas.
|
||||
|
||||
Let's consider a simple example:
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1"
|
||||
from langchain_core.messages import AnyMessage
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
||||
class State(TypedDict):
|
||||
messages: list[AnyMessage]
|
||||
extra_field: int
|
||||
```
|
||||
=== "Python"
|
||||
```python exec="on" source="above" session="1"
|
||||
from langchain_core.messages import AnyMessage
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
|
||||
class State(TypedDict):
|
||||
messages: list[AnyMessage]
|
||||
extra_field: int
|
||||
```
|
||||
=== "TypeScript"
|
||||
```typescript exec="1" source="above" session="1"
|
||||
import { BaseMessage } from "@langchain/core/messages";
|
||||
import { Annotation } from "@langchain/langgraph";
|
||||
|
||||
const StateAnnotation = Annotation.Root({
|
||||
messages: Annotation<BaseMessage[]>(),
|
||||
extraField: Annotation<number>(),
|
||||
});
|
||||
```
|
||||
|
||||
This state tracks a list of [message](https://python.langchain.com/docs/concepts/messages/) objects, as well as an extra integer field.
|
||||
|
||||
@@ -64,11 +80,12 @@ This node simply appends a message to our message list, and populates an extra f
|
||||
|
||||
!!! important
|
||||
|
||||
Nodes should return updates to the state directly, instead of mutating the state.
|
||||
```
|
||||
Nodes should return updates to the state directly, instead of mutating the state.
|
||||
```
|
||||
|
||||
Let's next define a simple graph containing this node. We use [StateGraph](../concepts/low_level.md#stategraph) to define a graph that operates on this state. We then use [add_node](../concepts/low_level.md#messagesstate) populate our graph.
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1"
|
||||
from langgraph.graph import StateGraph
|
||||
|
||||
@@ -80,7 +97,6 @@ graph = graph_builder.compile()
|
||||
|
||||
LangGraph provides built-in utilities for visualizing your graph. Let's inspect our graph. See [this guide](../how-tos/visualization.ipynb) for detail on visualization.
|
||||
|
||||
|
||||
```python
|
||||
from IPython.display import Image, display
|
||||
|
||||
@@ -95,7 +111,6 @@ In this case, our graph just executes a single node.
|
||||
|
||||
Let's proceed with a simple invocation:
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1" result="ansi"
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
@@ -110,7 +125,6 @@ Note that:
|
||||
|
||||
For convenience, we frequently inspect the content of [message objects](https://python.langchain.com/docs/concepts/messages/) via pretty-print:
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1" result="ansi"
|
||||
for message in result["messages"]:
|
||||
message.pretty_print()
|
||||
@@ -124,7 +138,6 @@ For `TypedDict` state schemas, we can define reducers by annotating the correspo
|
||||
|
||||
In the earlier example, our node updated the `"messages"` key in the state by appending a message to it. Below, we add a reducer to this key, such that updates are automatically appended:
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1"
|
||||
from typing_extensions import Annotated
|
||||
|
||||
@@ -142,7 +155,6 @@ class State(TypedDict):
|
||||
|
||||
Now our node can be simplified:
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1"
|
||||
def node(state: State):
|
||||
new_message = AIMessage("Hello!")
|
||||
@@ -150,7 +162,6 @@ def node(state: State):
|
||||
return {"messages": [new_message], "extra_field": 10}
|
||||
```
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1" result="ansi"
|
||||
from langgraph.graph import START
|
||||
|
||||
@@ -172,7 +183,6 @@ In practice, there are additional considerations for updating lists of messages:
|
||||
|
||||
LangGraph includes a built-in reducer `add_messages` that handles these considerations:
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1"
|
||||
from langgraph.graph.message import add_messages
|
||||
|
||||
@@ -191,7 +201,6 @@ def node(state: State):
|
||||
graph = StateGraph(State).add_node(node).set_entry_point("node").compile()
|
||||
```
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1" result="ansi"
|
||||
# highlight-next-line
|
||||
input_message = {"role": "user", "content": "Hi"}
|
||||
@@ -204,7 +213,6 @@ for message in result["messages"]:
|
||||
|
||||
This is a versatile representation of state for applications involving [chat models](https://python.langchain.com/docs/concepts/chat_models/). LangGraph includes a pre-built `MessagesState` for convenience, so that we can have:
|
||||
|
||||
|
||||
```python exec="on" source="above" session="1"
|
||||
from langgraph.graph import MessagesState
|
||||
|
||||
|
||||
+8
-1
@@ -2,11 +2,18 @@
|
||||
"name": "docs",
|
||||
"version": "1.0.0",
|
||||
"license": "MIT",
|
||||
"packageManager": "yarn@3.5.1",
|
||||
"scripts": {
|
||||
"build": "echo 'export OPENAI_API_KEY=\"sk-proj-1234567890\"' >> ~/.bashrc && echo 'export ANTHROPIC_API_KEY=\"sk-ant-api03-1234567890\"' >> ~/.bashrc && echo 'export PATH=$PATH:/vercel/.local/bin:$PATH' >> ~/.bashrc && source ~/.bashrc && make vercel-build-docs"
|
||||
"build": "make build-docs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@langchain/core": "^0.3.38",
|
||||
"@langchain/langgraph": "portal:./langgraphjs/libs/langgraph",
|
||||
"@langchain/langgraph-checkpoint": "portal:./langgraphjs/libs/checkpoint",
|
||||
"@langchain/langgraph-checkpoint-mongodb": "portal:./langgraphjs/libs/checkpoint-mongodb",
|
||||
"@langchain/langgraph-checkpoint-postgres": "portal:./langgraphjs/libs/checkpoint-postgres",
|
||||
"@langchain/langgraph-checkpoint-sqlite": "portal:./langgraphjs/libs/checkpoint-sqlite",
|
||||
"@langchain/langgraph-checkpoint-validation": "portal:./langgraphjs/libs/checkpoint-validation",
|
||||
"@langchain/openai": "^0.4.2",
|
||||
"msgpack-lite": "^0.1.26",
|
||||
"nock": "^14.0.1"
|
||||
|
||||
+3
-2
@@ -1,4 +1,5 @@
|
||||
{
|
||||
"buildCommand": "yarn build",
|
||||
"outputDirectory": "site"
|
||||
"buildCommand": "echo 'export OPENAI_API_KEY=\"sk-proj-1234567890\"' >> ~/.bashrc && echo 'export ANTHROPIC_API_KEY=\"sk-ant-api03-1234567890\"' >> ~/.bashrc && echo 'export PATH=$PATH:/vercel/.local/bin:$PATH' >> ~/.bashrc && source ~/.bashrc && make vercel-build-docs",
|
||||
"outputDirectory": "site",
|
||||
"installCommand": "echo done"
|
||||
}
|
||||
|
||||
+4823
-408
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user