Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
176020478c | ||
|
|
9ea6944dbf | ||
|
|
eb066bd31a | ||
|
|
6b83b9b019 | ||
|
|
9d68e5b3eb | ||
|
|
544386f6cc | ||
|
|
c2df8ab04b | ||
|
|
3cd57c038a | ||
|
|
4098bb138c | ||
|
|
6f874e9878 | ||
|
|
7d9a670710 | ||
|
|
c6b7486550 | ||
|
|
9c1a4efc96 | ||
|
|
283855b817 | ||
|
|
8c10fbdc04 | ||
|
|
a3ca41eb1b | ||
|
|
264535917f | ||
|
|
4de8792dc4 | ||
|
|
a24fc97778 |
No files matched your search
@@ -36,11 +36,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.4.22"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077, upload-time = "2026-04-22T11:26:11.191Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -40,11 +40,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.1.4"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -1364,6 +1364,17 @@ def test_search_empty_prefix_is_unconstrained(store: SqliteStore) -> None:
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batched", [False, True])
|
||||
def test_search_rejects_nul_namespace_prefix(store: SqliteStore, batched: bool) -> None:
|
||||
store.put(("a",), "key", {"v": 1})
|
||||
|
||||
with pytest.raises(InvalidNamespaceError, match="NUL"):
|
||||
if batched:
|
||||
store.batch([SearchOp(("a\x00",))])
|
||||
else:
|
||||
store.search(("a\x00",))
|
||||
|
||||
|
||||
def test_namespace_labels_with_trailing_newline(store: SqliteStore) -> None:
|
||||
"""Labels may contain newlines, and must not match a differently-named label.
|
||||
|
||||
|
||||
@@ -49,11 +49,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.1.4"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -1333,6 +1333,10 @@ def _validate_namespace_labels(namespace: tuple[str, ...]) -> None:
|
||||
raise InvalidNamespaceError(
|
||||
f"Invalid namespace label '{label}' found in {namespace}. Namespace labels cannot contain periods ('.')."
|
||||
)
|
||||
elif "\x00" in label:
|
||||
raise InvalidNamespaceError(
|
||||
f"Invalid namespace label '{label}' found in {namespace}. Namespace labels cannot contain NUL bytes ('\\x00')."
|
||||
)
|
||||
elif not label:
|
||||
raise InvalidNamespaceError(
|
||||
f"Namespace labels cannot be empty strings. Got {label} in {namespace}"
|
||||
|
||||
@@ -532,7 +532,7 @@ async def test_cannot_put_empty_namespace() -> None:
|
||||
assert (await async_store.aget(("valid", "namespace"), "key")) is None
|
||||
|
||||
|
||||
INVALID_NAMESPACES = [("foo.bar",), ("foo", ""), (123,)]
|
||||
INVALID_NAMESPACES = [("foo.bar",), ("foo\x00",), ("foo", ""), (123,)]
|
||||
NAMESPACE_METHODS = ["get", "delete", "search", "prefix", "suffix"]
|
||||
|
||||
|
||||
|
||||
@@ -51,11 +51,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.1.4"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -23,6 +23,7 @@ from langgraph_cli.schemas import (
|
||||
GraphDef,
|
||||
HttpConfig,
|
||||
IndexConfig,
|
||||
PriorityConfig,
|
||||
SecurityConfig,
|
||||
SerdeConfig,
|
||||
StoreConfig,
|
||||
@@ -113,6 +114,7 @@ def add_descriptions_to_schema(schema, cls):
|
||||
StoreConfig,
|
||||
IndexConfig,
|
||||
AuthConfig,
|
||||
PriorityConfig,
|
||||
SecurityConfig,
|
||||
HttpConfig,
|
||||
CorsConfig,
|
||||
|
||||
@@ -1776,17 +1776,17 @@ baseline-browser-mapping@^2.11.12:
|
||||
integrity sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw==
|
||||
|
||||
brace-expansion@^1.1.7:
|
||||
version "1.1.11"
|
||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
|
||||
integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==
|
||||
version "1.1.21"
|
||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.21.tgz#edf4fab5c64d051aea5a8def49aba1c7522279f3"
|
||||
integrity sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==
|
||||
dependencies:
|
||||
balanced-match "^1.0.0"
|
||||
concat-map "0.0.1"
|
||||
|
||||
brace-expansion@^2.0.2:
|
||||
version "2.0.2"
|
||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.2.tgz#54fc53237a613d854c7bd37463aad17df87214e7"
|
||||
integrity sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==
|
||||
version "2.1.7"
|
||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.1.7.tgz#14ea3836731823bb24740225ea43b941b436f9d1"
|
||||
integrity sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==
|
||||
dependencies:
|
||||
balanced-match "^1.0.0"
|
||||
|
||||
|
||||
@@ -581,9 +581,9 @@ base64-js@^1.5.1:
|
||||
integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
|
||||
|
||||
brace-expansion@^1.1.7:
|
||||
version "1.1.12"
|
||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.12.tgz#ab9b454466e5a8cc3a187beaad580412a9c5b843"
|
||||
integrity sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==
|
||||
version "1.1.21"
|
||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.21.tgz#edf4fab5c64d051aea5a8def49aba1c7522279f3"
|
||||
integrity sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==
|
||||
dependencies:
|
||||
balanced-match "^1.0.0"
|
||||
concat-map "0.0.1"
|
||||
|
||||
@@ -1 +1 @@
|
||||
__version__ = "0.4.37"
|
||||
__version__ = "0.4.38"
|
||||
@@ -847,6 +847,7 @@ def dev(
|
||||
store=config_json.get("store"),
|
||||
wait_for_client=wait_for_client,
|
||||
auth=config_json.get("auth"),
|
||||
priority=config_json.get("priority"),
|
||||
http=config_json.get("http"),
|
||||
ui=config_json.get("ui"),
|
||||
ui_config=config_json.get("ui_config"),
|
||||
|
||||
@@ -442,6 +442,7 @@ def validate_config(
|
||||
"store": config.get("store"),
|
||||
"auth": config.get("auth"),
|
||||
"encryption": config.get("encryption"),
|
||||
"priority": config.get("priority"),
|
||||
"http": config.get("http"),
|
||||
# Pass through webhooks config so it can be injected into the image
|
||||
"webhooks": config.get("webhooks"),
|
||||
@@ -606,6 +607,19 @@ def validate_config(
|
||||
f"Invalid encryption.path format: '{encryption_conf['path']}'. "
|
||||
"Must be in format './path/to/file.py:attribute_name'"
|
||||
)
|
||||
# Validate priority config
|
||||
if priority_conf := config.get("priority"):
|
||||
if "path" in priority_conf:
|
||||
if ":" not in priority_conf["path"]:
|
||||
raise ValueError(
|
||||
f"Invalid priority.path format: '{priority_conf['path']}'. "
|
||||
"Must be in format './path/to/file.py:attribute_name'"
|
||||
)
|
||||
if _is_node_graph(priority_conf["path"]):
|
||||
raise ValueError(
|
||||
"JavaScript priority callbacks are not supported: "
|
||||
f"'{priority_conf['path']}'. Use a Python file or module instead."
|
||||
)
|
||||
if http_conf := config.get("http"):
|
||||
if "app" in http_conf:
|
||||
if ":" not in http_conf["app"]:
|
||||
@@ -649,6 +663,7 @@ _KNOWN_CONFIG_KEYS = {
|
||||
"store",
|
||||
"auth",
|
||||
"encryption",
|
||||
"priority",
|
||||
"http",
|
||||
"webhooks",
|
||||
"checkpointer",
|
||||
@@ -1080,6 +1095,49 @@ def _update_encryption_path(
|
||||
)
|
||||
|
||||
|
||||
def _update_priority_path(
|
||||
config_path: pathlib.Path, config: Config, local_deps: LocalDeps
|
||||
) -> None:
|
||||
"""Update priority.path to use Docker container paths."""
|
||||
priority_conf = config.get("priority")
|
||||
if not priority_conf or not (path_str := priority_conf.get("path")):
|
||||
return
|
||||
|
||||
module_str, sep, attr_str = path_str.partition(":")
|
||||
if not sep or not module_str.startswith("."):
|
||||
return # Already validated or absolute path
|
||||
|
||||
resolved = config_path.parent / module_str
|
||||
if not resolved.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Priority file not found: {resolved} (from {path_str})"
|
||||
)
|
||||
if not resolved.is_file():
|
||||
raise IsADirectoryError(f"Priority path must be a file: {resolved}")
|
||||
|
||||
# Check faux packages first (they take precedence)
|
||||
for faux_path, (_, destpath) in local_deps.faux_pkgs.items():
|
||||
if resolved.is_relative_to(faux_path):
|
||||
new_path = f"{destpath}/{resolved.relative_to(faux_path)}:{attr_str}"
|
||||
priority_conf["path"] = new_path
|
||||
return
|
||||
|
||||
# Check real packages
|
||||
for real_path in local_deps.real_pkgs:
|
||||
if resolved.is_relative_to(real_path):
|
||||
new_path = (
|
||||
f"/deps/{real_path.name}/{resolved.relative_to(real_path)}:{attr_str}"
|
||||
)
|
||||
priority_conf["path"] = new_path
|
||||
return
|
||||
|
||||
raise ValueError(
|
||||
f"Priority file '{resolved}' not covered by dependencies.\n"
|
||||
"Add its parent directory to the 'dependencies' array in your config.\n"
|
||||
f"Current dependencies: {config['dependencies']}"
|
||||
)
|
||||
|
||||
|
||||
def _update_checkpointer_path(
|
||||
config_path: pathlib.Path, config: Config, local_deps: LocalDeps
|
||||
) -> None:
|
||||
@@ -1213,6 +1271,9 @@ def _build_runtime_env_vars(config: Config) -> list[str]:
|
||||
if (encryption_config := config.get("encryption")) is not None:
|
||||
env_vars.append(f"ENV LANGGRAPH_ENCRYPTION='{json.dumps(encryption_config)}'")
|
||||
|
||||
if (priority_config := config.get("priority")) is not None:
|
||||
env_vars.append(f"ENV LANGGRAPH_PRIORITY='{json.dumps(priority_config)}'")
|
||||
|
||||
if (http_config := config.get("http")) is not None:
|
||||
env_vars.append(f"ENV LANGGRAPH_HTTP='{json.dumps(http_config)}'")
|
||||
|
||||
@@ -1380,6 +1441,8 @@ def python_config_to_docker(
|
||||
_update_auth_path(config_path, config, local_deps)
|
||||
# Rewrite encryption path, so it points to the correct location in the Docker container
|
||||
_update_encryption_path(config_path, config, local_deps)
|
||||
# Rewrite priority path, so it points to the correct location in the Docker container
|
||||
_update_priority_path(config_path, config, local_deps)
|
||||
# Rewrite checkpointer path, so it points to the correct location in the Docker container
|
||||
_update_checkpointer_path(config_path, config, local_deps)
|
||||
# Rewrite HTTP app path, so it points to the correct location in the Docker container
|
||||
|
||||
@@ -373,6 +373,27 @@ class EncryptionConfig(TypedDict, total=False):
|
||||
"""
|
||||
|
||||
|
||||
class PriorityConfig(TypedDict, total=False):
|
||||
"""Configuration for custom run priority logic.
|
||||
|
||||
Allows you to decide each thread's priority settings with a Python function
|
||||
that the server calls at the first run on a thread with no settings, which
|
||||
can come long after the thread was created."""
|
||||
|
||||
path: str
|
||||
"""Required. Path to an instance of the Priority() class that registers an assign function.
|
||||
|
||||
Format: "path/to/file.py:my_priority"
|
||||
|
||||
Example:
|
||||
{
|
||||
"priority": {
|
||||
"path": "./priority.py:my_priority"
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
class CorsConfig(TypedDict, total=False):
|
||||
"""Specifies Cross-Origin Resource Sharing (CORS) rules for your server.
|
||||
|
||||
@@ -753,6 +774,13 @@ class Config(TypedDict, total=False):
|
||||
Allows you to implement custom encryption for sensitive data stored in the database.
|
||||
"""
|
||||
|
||||
priority: PriorityConfig | None
|
||||
"""Optional. Custom run priority config, including the path to your Python priority logic.
|
||||
|
||||
Allows you to decide each thread's priority settings at the first run on a thread with no settings,
|
||||
which can come long after the thread was created.
|
||||
"""
|
||||
|
||||
http: HttpConfig | None
|
||||
"""Optional. Configuration for the built-in HTTP server, controlling which custom routes are exposed
|
||||
and how cross-origin requests are handled.
|
||||
@@ -784,6 +812,7 @@ __all__ = [
|
||||
"CheckpointerConfig",
|
||||
"AuthConfig",
|
||||
"EncryptionConfig",
|
||||
"PriorityConfig",
|
||||
"HttpConfig",
|
||||
"MiddlewareOrders",
|
||||
"Distros",
|
||||
|
||||
@@ -910,6 +910,7 @@ def python_config_to_docker_uv_lock(
|
||||
for section, key in [
|
||||
("auth", "path"),
|
||||
("encryption", "path"),
|
||||
("priority", "path"),
|
||||
("checkpointer", "path"),
|
||||
("http", "app"),
|
||||
]:
|
||||
|
||||
@@ -196,6 +196,17 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"priority": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/PriorityConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional. Custom run priority config, including the path to your Python priority logic.\n\nAllows you to decide each thread's priority settings at the first run on a thread with no settings,\nwhich can come long after the thread was created.\n"
|
||||
},
|
||||
"source": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -438,6 +449,17 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"priority": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/PriorityConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional. Custom run priority config, including the path to your Python priority logic.\n\nAllows you to decide each thread's priority settings at the first run on a thread with no settings,\nwhich can come long after the thread was created.\n"
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/$defs/UvSource"
|
||||
},
|
||||
@@ -666,6 +688,17 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"priority": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/PriorityConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional. Custom run priority config, including the path to your Python priority logic.\n\nAllows you to decide each thread's priority settings at the first run on a thread with no settings,\nwhich can come long after the thread was created.\n"
|
||||
},
|
||||
"source": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -1170,6 +1203,18 @@
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"PriorityConfig": {
|
||||
"title": "PriorityConfig",
|
||||
"description": "Configuration for custom run priority logic.\n\nAllows you to decide each thread's priority settings with a Python function\nthat the server calls at the first run on a thread with no settings, which\ncan come long after the thread was created.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string",
|
||||
"description": "Required. Path to an instance of the Priority() class that registers an assign function.\n"
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"UvSource": {
|
||||
"title": "UvSource",
|
||||
"description": "Deployment source rooted at a uv project or workspace.",
|
||||
|
||||
@@ -196,6 +196,17 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"priority": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/PriorityConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional. Custom run priority config, including the path to your Python priority logic.\n\nAllows you to decide each thread's priority settings at the first run on a thread with no settings,\nwhich can come long after the thread was created.\n"
|
||||
},
|
||||
"source": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -438,6 +449,17 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"priority": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/PriorityConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional. Custom run priority config, including the path to your Python priority logic.\n\nAllows you to decide each thread's priority settings at the first run on a thread with no settings,\nwhich can come long after the thread was created.\n"
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/$defs/UvSource"
|
||||
},
|
||||
@@ -666,6 +688,17 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"priority": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/PriorityConfig"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional. Custom run priority config, including the path to your Python priority logic.\n\nAllows you to decide each thread's priority settings at the first run on a thread with no settings,\nwhich can come long after the thread was created.\n"
|
||||
},
|
||||
"source": {
|
||||
"anyOf": [
|
||||
{
|
||||
@@ -1170,6 +1203,18 @@
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"PriorityConfig": {
|
||||
"title": "PriorityConfig",
|
||||
"description": "Configuration for custom run priority logic.\n\nAllows you to decide each thread's priority settings with a Python function\nthat the server calls at the first run on a thread with no settings, which\ncan come long after the thread was created.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string",
|
||||
"description": "Required. Path to an instance of the Priority() class that registers an assign function.\n"
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"UvSource": {
|
||||
"title": "UvSource",
|
||||
"description": "Deployment source rooted at a uv project or workspace.",
|
||||
|
||||
@@ -2,10 +2,12 @@ import json
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import click
|
||||
from click.testing import CliRunner
|
||||
@@ -371,6 +373,32 @@ def test_dev_command_requires_ssl_certfile_and_keyfile_together(tmp_path) -> Non
|
||||
assert "Both --ssl-certfile and --ssl-keyfile must be provided" in result.output
|
||||
|
||||
|
||||
def test_dev_command_passes_priority_to_the_server(tmp_path, monkeypatch) -> None:
|
||||
config_path = tmp_path / "langgraph.json"
|
||||
config_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"dependencies": ["."],
|
||||
"graphs": {"agent": "./agent.py:graph"},
|
||||
"priority": {"path": "./priority.py:priority"},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# The test environment doesn't install langgraph-api, so stand in for its server.
|
||||
server = MagicMock()
|
||||
monkeypatch.setitem(sys.modules, "langgraph_api.cli", server)
|
||||
# dev appends the working directory and its dependencies to sys.path.
|
||||
monkeypatch.setattr(sys, "path", list(sys.path))
|
||||
|
||||
result = CliRunner().invoke(cli, ["dev", "--config", str(config_path)])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert server.run_server.call_args.kwargs["priority"] == {
|
||||
"path": "./priority.py:priority"
|
||||
}
|
||||
|
||||
|
||||
def test_deploy_list_command(monkeypatch) -> None:
|
||||
runner = CliRunner()
|
||||
captured: dict[str, str] = {}
|
||||
|
||||
@@ -16,6 +16,7 @@ from langgraph_cli.config import (
|
||||
config_to_docker,
|
||||
default_base_image,
|
||||
docker_tag,
|
||||
get_unknown_keys,
|
||||
has_disallowed_build_command_content,
|
||||
validate_config,
|
||||
validate_config_file,
|
||||
@@ -155,6 +156,7 @@ def test_validate_config():
|
||||
"store": None,
|
||||
"auth": None,
|
||||
"encryption": None,
|
||||
"priority": None,
|
||||
"webhooks": None,
|
||||
"checkpointer": None,
|
||||
"http": None,
|
||||
@@ -184,6 +186,7 @@ def test_validate_config():
|
||||
"store": None,
|
||||
"auth": None,
|
||||
"encryption": None,
|
||||
"priority": None,
|
||||
"webhooks": None,
|
||||
"checkpointer": None,
|
||||
"http": None,
|
||||
@@ -1289,6 +1292,109 @@ def test_config_to_docker_python_encryption_formatted():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["./priority.py:priority", "my_package.priority:priority"]
|
||||
)
|
||||
def test_config_to_docker_python_priority(path: str):
|
||||
# Test that priority config is kept by validation and isn't an unknown key
|
||||
raw_config = {
|
||||
"python_version": "3.11",
|
||||
"graphs": {"agent": "./agent.py:graph"},
|
||||
"dependencies": ["."],
|
||||
"priority": {"path": path},
|
||||
}
|
||||
validated = validate_config(raw_config)
|
||||
|
||||
assert validated["priority"] == {"path": path}
|
||||
assert get_unknown_keys(raw_config) == []
|
||||
|
||||
|
||||
def test_config_to_docker_python_priority_bad_path():
|
||||
# Test that invalid priority path format raises ValueError
|
||||
with pytest.raises(ValueError, match="Invalid priority.path format"):
|
||||
validate_config(
|
||||
{
|
||||
"python_version": "3.11",
|
||||
"graphs": {"agent": "./agent.py:graph"},
|
||||
"dependencies": ["."],
|
||||
"priority": {"path": "./priority.py"}, # Missing :attribute
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ext", [".ts", ".mts", ".cts", ".js", ".mjs", ".cjs"])
|
||||
def test_config_to_docker_python_priority_js_path(ext: str):
|
||||
# Test that a JavaScript priority path is rejected
|
||||
with pytest.raises(ValueError, match="JavaScript priority callbacks"):
|
||||
validate_config(
|
||||
{
|
||||
"python_version": "3.11",
|
||||
"graphs": {"agent": "./agent.py:graph"},
|
||||
"dependencies": ["."],
|
||||
"priority": {"path": f"./priority{ext}:priority"},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_config_to_docker_nodejs_priority():
|
||||
# Test that a Node.js-only project can name a Python priority file
|
||||
actual_docker_stdin, _ = config_to_docker(
|
||||
PATH_TO_CONFIG,
|
||||
validate_config(
|
||||
{
|
||||
"node_version": "20",
|
||||
"graphs": {"agent": "./graphs/agent.js:graph"},
|
||||
"priority": {"path": "./priority.py:priority"},
|
||||
}
|
||||
),
|
||||
base_image="langchain/langgraphjs-api",
|
||||
)
|
||||
assert (
|
||||
"""ENV LANGGRAPH_PRIORITY='{"path": "./priority.py:priority"}'"""
|
||||
in actual_docker_stdin
|
||||
)
|
||||
|
||||
|
||||
def test_config_to_docker_python_priority_formatted():
|
||||
# Test that priority config is written to the Docker output with its container path
|
||||
actual_docker_stdin, _ = config_to_docker(
|
||||
PATH_TO_CONFIG,
|
||||
validate_config(
|
||||
{
|
||||
"python_version": "3.11",
|
||||
"dependencies": ["."],
|
||||
"graphs": {"agent": "./graphs/agent.py:graph"},
|
||||
"priority": {"path": "./agent.py:my_priority"},
|
||||
}
|
||||
),
|
||||
base_image="langchain/langgraph-api",
|
||||
)
|
||||
assert (
|
||||
"ENV LANGGRAPH_PRIORITY="
|
||||
"""'{"path": "/deps/outer-unit_tests/unit_tests/agent.py:my_priority"}'"""
|
||||
in actual_docker_stdin
|
||||
)
|
||||
|
||||
|
||||
def test_config_to_docker_python_priority_not_in_dependencies():
|
||||
# Test that a priority file outside the dependencies raises ValueError
|
||||
with pytest.raises(
|
||||
ValueError, match="Priority file .* not covered by dependencies"
|
||||
):
|
||||
config_to_docker(
|
||||
PATH_TO_CONFIG,
|
||||
validate_config(
|
||||
{
|
||||
"python_version": "3.11",
|
||||
"dependencies": ["./graphs"],
|
||||
"graphs": {"agent": "./graphs/agent.py:graph"},
|
||||
"priority": {"path": "./agent.py:my_priority"},
|
||||
}
|
||||
),
|
||||
base_image="langchain/langgraph-api",
|
||||
)
|
||||
|
||||
|
||||
def test_config_to_docker_nodejs_internal_docker_tag():
|
||||
graphs = {"agent": "./graphs/agent.js:graph"}
|
||||
actual_docker_stdin, additional_contexts = config_to_docker(
|
||||
@@ -1731,6 +1837,38 @@ def test_config_to_docker_uv_lock_honors_root_workspace_sources():
|
||||
)
|
||||
|
||||
|
||||
def test_config_to_docker_uv_lock_rewrites_priority_path():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmpdir_path = pathlib.Path(tmpdir)
|
||||
project_root, config_path = _write_uv_lock_workspace(
|
||||
tmpdir_path,
|
||||
agent_sources="[tool.uv.sources]\nshared = { workspace = true }",
|
||||
)
|
||||
(project_root / "libs" / "shared" / "src" / "shared" / "priority.py").touch()
|
||||
|
||||
config = validate_config(
|
||||
{
|
||||
"python_version": "3.11",
|
||||
"graphs": {
|
||||
"agent": "../../apps/agent/src/agent/graph.py:graph",
|
||||
},
|
||||
"source": {"kind": "uv", "root": "../..", "package": "agent"},
|
||||
"priority": {
|
||||
"path": "../../libs/shared/src/shared/priority.py:priority"
|
||||
},
|
||||
}
|
||||
)
|
||||
docker, _ = config_to_docker(
|
||||
config_path, config, base_image="langchain/langgraph-api:0.2.47"
|
||||
)
|
||||
|
||||
assert (
|
||||
"ENV LANGGRAPH_PRIORITY="
|
||||
"""'{"path": "/deps/workspace/libs/shared/src/shared/priority.py:priority"}'"""
|
||||
in docker
|
||||
)
|
||||
|
||||
|
||||
def test_config_to_docker_uv_lock_ignores_unrelated_workspace_package_sources():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmpdir_path = pathlib.Path(tmpdir)
|
||||
|
||||
@@ -52,11 +52,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.2.25"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -26,11 +26,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.5.20"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -168,11 +168,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.2.25"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2292,28 +2292,28 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "uv"
|
||||
version = "0.11.15"
|
||||
version = "0.13.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/da/34/609d5d01ba21dc8f0974610ca7802fbb2c946a0c38665cfe5c5aeddbefb5/uv-0.11.15.tar.gz", hash = "sha256:755f959ec6a2fd8ccb6ee76ad90ab759d2eb1f4797444078645dd1ee4bca92d6", size = 4159545, upload-time = "2026-05-18T19:57:48.133Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/79/55/609455d21e87430c11fe734abfced50a3bdf177b2f49d41fbb19c199cbc3/uv-0.13.4.tar.gz", hash = "sha256:b0bff5b9a5061f7a35b1d0c1a94c19af134222c48eb7aadc42f85a038be1b248", size = 7482486, upload-time = "2026-10-20T15:12:06.801Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/7c/dcc230c5911884d8848145dabcac8fb95a5ed6f9fe1c57fae8242618f28a/uv-0.11.15-py3-none-linux_armv6l.whl", hash = "sha256:83b04ab49514a0a761ffedb36a748ee81f87746671e72088e5f32c9585e5f1a9", size = 23110183, upload-time = "2026-05-18T19:57:23.051Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/f3/efd4e044b60eb9c3c12ee386be098d56c335538ccec7caa49349cfba9344/uv-0.11.15-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b6cae61f737be075b90be9e3f07d961072aed7019f4c9b8ed5c5d41c4d6cade3", size = 22637941, upload-time = "2026-05-18T19:57:26.752Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/b8/48627f895a1569e576822e0a8416aa4797eb4a4551de21a4ad97b9b5819d/uv-0.11.15-py3-none-macosx_11_0_arm64.whl", hash = "sha256:9accae33619a9166e5c48531deb455d672cfb89f9357a00975e669c76b0bd49f", size = 21258803, upload-time = "2026-05-18T19:57:05.473Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/af/50/4bc8a148274feabee2d9c9f1fa15009e10c0228dfe57981ee3ea2ef1d481/uv-0.11.15-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:c0cf52cd6d50bb9e05e2d968f45f80761107e4cbc8d4a26d9758f9d8274aaec1", size = 23066178, upload-time = "2026-05-18T19:57:33.058Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/56/139fc3bec9a8b0a25bfe2196123adb9f16124da437bf4fbcf0d21cfcafb2/uv-0.11.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:49dc6ed70bff00937384f96cdc4b1a4742d18e5504ec2c4a1214dba2dee5687a", size = 22705332, upload-time = "2026-05-18T19:57:36.714Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/b0/b18b3dd204f8c213236a1ebd148e009861637129a8cce34df0e9aa22ed40/uv-0.11.15-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:adb9a89352539fdd8f7cd5f9966cf9f94fc5b98e0ccdf5003a04123dc6423bec", size = 22707534, upload-time = "2026-05-18T19:58:04.117Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/76/36/3ca09f95572df99d361b49c96b1297149e96e120d8d1ecf074095a4b6da4/uv-0.11.15-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40ff67e3f8e8a7533781a2e892a534975a93acb83ea35460e64e7b2bf2111774", size = 24096607, upload-time = "2026-05-18T19:58:11.625Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/be/3bdee21a296bbf5336a526e3613d0e7d4538dacc39c62d7fcba55d15f6b0/uv-0.11.15-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6463a299ed7e6b5a800ed6f108af8e1588352629424133ddef7572b0e1e1118", size = 25082562, upload-time = "2026-05-18T19:57:40.69Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cd/73/f371f3689ffe741066468d001d85f739fc4b5574de83b639ef19b5e8a7f4/uv-0.11.15-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68c1e62d4b78578b90b833553286b65d6a7e327537716441068583ba652ec4f5", size = 24253391, upload-time = "2026-05-18T19:57:18.47Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d3/16/fe392d618af6b00c064b3e718d585dcf791546a77c5123a5bec07ce53a0a/uv-0.11.15-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98edf1bdaf82447014852051d93e3ee95012509c567bf057fd117e6bdbd9a807", size = 24415871, upload-time = "2026-05-18T19:58:19.651Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/24/2e92a052fb6334fcd746d1c7cb57847c204b118c84f5da53c0f9e129f7b7/uv-0.11.15-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:be8f76d25bcf4c92bb384240ac1bf9aa7f51063d0bdeca4c9cf0ec3ed8b145e0", size = 23159007, upload-time = "2026-05-18T19:57:10.653Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/2e/6923d0658d164bb2c435ed1868aa2d49b3074594679917a001ff92dc95bb/uv-0.11.15-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:f9f4fbbf4fe485522054f3c7496c6e8e932d6436e4200ff3daf718db0b7c7bd5", size = 23769385, upload-time = "2026-05-18T19:58:15.856Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/99/7e34cd949e57360814e8064cc9fb7104df445d0f6a663504e5f7473480aa/uv-0.11.15-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:0ed920e896b2fd13a35031707e307e42fbb2681458b967440a17272d86d49137", size = 23860973, upload-time = "2026-05-18T19:57:55.575Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/28/98/8fe1f5f9d816e94569a0298dd8e0936801097625fa1952162951f0d628b6/uv-0.11.15-py3-none-musllinux_1_1_i686.whl", hash = "sha256:41d907611f3e6a13262807fd7f0a17849f76285ca80f536f6b3943732bdc6656", size = 23431392, upload-time = "2026-05-18T19:57:59.814Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cc/6b/76a1ce2fa860026913a5941700cdc7d715fce9c3277a3fa3489cf2523ca0/uv-0.11.15-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:e3b68f8bf1a4568710f77e5bda9182ce7682811d89a8e7468c22460e032b234d", size = 24519478, upload-time = "2026-05-18T19:57:51.165Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/43/60/1d58e8a05718cb50494763115710b73846cacb651fd735d285233fd72c59/uv-0.11.15-py3-none-win32.whl", hash = "sha256:8e2da3076761086a5b76869c3f38ef0509c836046ef41ddd19485dfd7271dca9", size = 22020178, upload-time = "2026-05-18T19:58:07.64Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/53/40fcefcb348af660488597ed3c01363df7344e60611f8883750dc596f5c6/uv-0.11.15-py3-none-win_amd64.whl", hash = "sha256:cc3915ab291a1ecaf31de05f5d8bd70d09c66fe9911a53f70d9efa62ff0dbd8a", size = 24668779, upload-time = "2026-05-18T19:57:44.894Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/7d/fa3a9960c95af9bbe2a629048760d0b9b4fead8ccd4f2235af747ec7cdf0/uv-0.11.15-py3-none-win_arm64.whl", hash = "sha256:4f39426a13dee24897aed60c4b98058c66f18bd983885ac5f4a54a04b24fbddf", size = 23198178, upload-time = "2026-05-18T19:57:14.68Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/bd/4349da5356b17a1c0e36c8552c809925f86f6292de9b1f599f01e8e8093d/uv-0.13.4-py3-none-linux_armv6l.whl", hash = "sha256:205a4f93e568fb24f45a2f3834f1c0a8f1adb5f396bf473c71030ed707dce048", size = 18272390, upload-time = "2026-10-20T15:11:02.468Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/f2/0cb77a1ae43959772f8f7873eb1f4061907fbb1694ce55c79b231f639b1d/uv-0.13.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b4bbacea78c9db136cd2ec3147cbfd9ed910b4db5e1877109bd74d0e73bb2eaf", size = 17384113, upload-time = "2026-10-20T15:11:07.172Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/26/44/5a8733dd1b2e8d6f9cffd1ff6428a3e11dcc7e146a7657e32cfb8d9fb4a4/uv-0.13.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:95f4eb2b991b53b50ec2b18ab612fca8ba1e7a75a47af0331fb8bcd21da3eb03", size = 14433723, upload-time = "2026-10-20T15:11:10.185Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/39/be14f051bf34a1822d634819890a2d0cba3b69101e633fb7b9f54d7b4020/uv-0.13.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:e90ab42825587821096cb0d071e387468da38cfa22818c91f9c0b3c0453048e8", size = 17816443, upload-time = "2026-10-20T15:11:13.674Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/97/a0/5df0f8bc6367dd37b98432a7c462e2918554f6db2a71211574852b8f0727/uv-0.13.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:3a9dcb6762c3ba249d8c084315ba03839082af3ece3536cc4164ec735c351d68", size = 17828025, upload-time = "2026-10-20T15:11:16.806Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/9c/2fed5357700b7c304b94214979e4efe680c3bc6e337a97eedaa372fe0947/uv-0.13.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ae6ff2b07892379dc767c9064c2b8bbfac88e767edac7ce28e1b0f15d86abb4", size = 17843737, upload-time = "2026-10-20T15:11:20.043Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4c/2e/ee026284f096236dc3e1abed0316f9edab65c0ea176ea0072c064e62478d/uv-0.13.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25cc14ff4fd8335d3be2cc493a621a003fdafcb0fb1c953c905daabe4def5ab8", size = 18576350, upload-time = "2026-10-20T15:11:24.035Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6d/f1/67fbdb649c8c5d0cb38ea74829080479b77c8631cda5d77fc1b28f027ca7/uv-0.13.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:77062b02856c74697f48328d8a6583ab07e6da0a993e86c2db46f3d1da27c637", size = 19831234, upload-time = "2026-10-20T15:11:27.627Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bd/00/7d49e3a1a3e98e5244fa38e0c964dc8de568bf3bb101257a63dc2602efed/uv-0.13.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fb0e2028d9a8929884344afb7d91d0f65d9783f640d28aede89fca3602bedde9", size = 19702886, upload-time = "2026-10-20T15:11:31.128Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/a1/de02c0913726728eca56ee258875cbba9c7b0572c63f494fd9f55c7852e3/uv-0.13.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05e4332791b4d55b6dde8695b7fa6079b26991519f44760887c5dfbd5ca4fe53", size = 17181168, upload-time = "2026-10-20T15:11:34.583Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a8/cd/f4a3c7377674366798c56aabd0f9a1f05687c3939f5ba857a3ea6093ff84/uv-0.13.4-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:7b3cdf868e5415631fe797b648c0d6945d38d388f23367bdd0b6332e80a1fcb0", size = 16502020, upload-time = "2026-10-20T15:11:37.88Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/f4/ce79657c5a0f51b602db9a663529baafff03b9fe8cc41e09ad3f7246b88a/uv-0.13.4-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:4e6b08578c920b86910897f36bb6da7229c8e7e0187976570e2a538cb28b777e", size = 18000980, upload-time = "2026-10-20T15:11:41.59Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/21/e6/7d77022d5661f8e01578bc5423bf02af408220e7e7ac5ee15eb94742c7cb/uv-0.13.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:d86a5dec8ab30c70519c71ca5085c8aaf28664836cbc600951cedaa3fc678cbf", size = 18179083, upload-time = "2026-10-20T15:11:45.191Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/40/1bf0324800d38009774ba087a6fbffddc76a3e5bf42c081b9298b1236fb5/uv-0.13.4-py3-none-musllinux_1_1_i686.whl", hash = "sha256:5d89d1028c2e22977d9215cbb79cfe12559cb0c8a507a12e2f3a7e47930cf1bf", size = 18010389, upload-time = "2026-10-20T15:11:48.963Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/1e/ec/f352ca2cc3b4b6191bb4ec8cdf4bc190eed0ba8bda4c33b7fd66c5d33cec/uv-0.13.4-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:52b5c753a7c2d5f8323a70bb4c93118988356331d0758afa6446c81f974e7196", size = 19004998, upload-time = "2026-10-20T15:11:52.628Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/4f/fc05ff4292edfe785b47874ec86d3d83ca2bf735cff8a9d2170ffb66ad83/uv-0.13.4-py3-none-win32.whl", hash = "sha256:d333f01852b817ba05612de31474739da1af7e5463d119b30a9c4331eba1f408", size = 16190502, upload-time = "2026-10-20T15:11:56.657Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/2f/b9/b526eab6c72eacc705a58378f6850fde046b28c86fed01bd42da61367c6b/uv-0.13.4-py3-none-win_amd64.whl", hash = "sha256:b2ff80397273fff0e02c20eacae6882834c811827d6fc08f658db156fe754386", size = 15331833, upload-time = "2026-10-20T15:11:59.733Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7c/f7/4d5fa89d3d10ff6a84e547ef6e02c45b2f2d50e66682da778acf60db63f1/uv-0.13.4-py3-none-win_arm64.whl", hash = "sha256:deffa5a4afdb48f3e44fb26e80377d256b330ddcad0373daff4b54f6162b43a4", size = 15690151, upload-time = "2026-10-20T15:12:02.74Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
# Distributed DeltaChannel benchmark
|
||||
|
||||
The coordinator (`delta_distributed.py`) creates one disposable LangSmith sandbox
|
||||
per `(frequency, payload)` pair: 40 shards for the default 25–500 step-25 grid.
|
||||
Each shard owns a local Postgres process and grows one graph to 10000 updates,
|
||||
recording the five prefix lengths. `GenericFakeChatModel` scripts are deterministic;
|
||||
no provider API is used. Parallelism changes wall time, not total benchmark work.
|
||||
|
||||
## Snapshot contract
|
||||
|
||||
Supply an existing, immutable LangSmith snapshot with:
|
||||
|
||||
- Linux, Python 3.14, git, PostgreSQL 14 binaries (`initdb`, `pg_ctl`) on PATH;
|
||||
- a checkout at `/workspace/langgraph`, pinned to the intended revision;
|
||||
- dependencies installed using `uv sync --frozen --group dev` in
|
||||
`/workspace/langgraph/libs/langgraph`;
|
||||
- `/workspace/langgraph/libs/langgraph/.venv/bin/python` available;
|
||||
- a writable `/tmp`; a non-root user able to run Postgres, or root plus `runuser`
|
||||
and the `postgres` OS account;
|
||||
- no Postgres listener already occupying loopback port 5432;
|
||||
- **no secrets baked into the snapshot**. Credentials are needed only by the
|
||||
coordinator, not the shard. Use a dedicated snapshot, not a personal workspace
|
||||
image with unrelated processes or credentials.
|
||||
|
||||
The coordinator uploads only the benchmark source and shard worker. Libraries,
|
||||
interpreter and dependency versions come from the supplied snapshot. Every job
|
||||
uses that same snapshot ID and explicit CPU/memory/disk configuration. Snapshot
|
||||
construction/provisioning is intentionally separate; no image or paid fleet has
|
||||
been created by this change. This is a normal standalone benchmark project using
|
||||
the LangSmith SDK, not a change to the Open SWE workspace image.
|
||||
|
||||
## Create the snapshot
|
||||
|
||||
`delta_snapshot.py` builds the required image from the dedicated
|
||||
`delta-snapshot/Dockerfile` context. Only that Dockerfile is uploaded, never your
|
||||
working directory, `.env`, git credentials or LangSmith key. It uses Postgres
|
||||
14.7, uv 0.13.4 and Python 3.14.3, checks out a full public LangGraph commit SHA,
|
||||
and syncs the frozen dev dependencies. OS packages and version-tagged base images
|
||||
are not digest/apt-repository pinned; reuse the resulting immutable snapshot ID
|
||||
for comparable measurements rather than rebuilding between shards.
|
||||
|
||||
From `libs/langgraph`, after checking out this PR and syncing dependencies:
|
||||
|
||||
```bash
|
||||
uv run python bench/delta_snapshot.py \
|
||||
--name delta-bench --output ./snapshot-plan.json
|
||||
```
|
||||
|
||||
This dry run creates no resources. By default, the revision is local HEAD: it must
|
||||
already be pushed and reachable in the public repository. Uncommitted local code
|
||||
is **not** included. For a paid build, use a fresh receipt path:
|
||||
|
||||
```bash
|
||||
uv run python bench/delta_snapshot.py \
|
||||
--name delta-bench --output ./snapshot.json --execute
|
||||
```
|
||||
|
||||
Optional flags: `--revision FULL_40_CHAR_SHA --vcpus 2 --memory-gib 8 --disk-gib 30
|
||||
--timeout 1800`. The defaults are builder sizing, not fleet concurrency. The SDK
|
||||
creates one temporary billable builder sandbox and a persistent snapshot, using
|
||||
`LANGSMITH_API_KEY`/`LANGSMITH_ENDPOINT` only on the coordinator. It cleans up the
|
||||
builder when the call exits; verify cleanup in LangSmith after a crash or failed
|
||||
build. The snapshot remains until you explicitly delete it. This command does not
|
||||
launch benchmark shards or alter an Open SWE workspace image.
|
||||
|
||||
On success, `snapshot.json` contains `snapshot_id`, revision and build parameters:
|
||||
|
||||
```bash
|
||||
export SNAPSHOT_ID="$(python -c 'import json; print(json.load(open("snapshot.json"))["snapshot_id"])')"
|
||||
```
|
||||
|
||||
Then use `--snapshot-id "$SNAPSHOT_ID"` in the pilot command below. Build failures
|
||||
leave a `building` receipt: inspect LangSmith before retrying with a new name/path;
|
||||
do not assume a timeout means no snapshot or builder was created. Remote image
|
||||
build and pilot have not yet been executed; local SDK-contract tests are not a
|
||||
substitute for that end-to-end verification.
|
||||
|
||||
## Plan first
|
||||
|
||||
From `libs/langgraph`:
|
||||
|
||||
```bash
|
||||
uv run python bench/delta_distributed.py \
|
||||
--snapshot-id YOUR_SNAPSHOT_ID --concurrency 4 --timeout 14400 \
|
||||
--vcpus 2 --memory-gib 8 --disk-gib 30 --output /tmp/delta-fleet
|
||||
```
|
||||
|
||||
Without `--execute`, this only writes `plan.json`; it does not instantiate an SDK
|
||||
client, require credentials or provision resources. Values above are **examples,
|
||||
not selected budgets**. Agree on them before executing. With 40 shards and a
|
||||
4-hour benchmark timeout, the benchmark allowance alone is 160 sandbox-hours;
|
||||
readiness and result collection add overhead. The printed allowance is an
|
||||
estimate, not a monetary billing cap. Concurrency is capped at 40 and each shard
|
||||
benchmark timeout at 24 hours. There is no automatic retry storm.
|
||||
|
||||
Start with a small pilot (new output directory):
|
||||
|
||||
```bash
|
||||
uv run python bench/delta_distributed.py \
|
||||
--snapshot-id YOUR_SNAPSHOT_ID --concurrency 2 --timeout 900 \
|
||||
--vcpus 2 --memory-gib 8 --disk-gib 30 --output /tmp/delta-pilot \
|
||||
--frequencies 25 300 --lengths 50 200 --profiles mock700
|
||||
```
|
||||
|
||||
Review the generated plan, then repeat the exact command with `--execute` to
|
||||
create billable resources. The SDK reads `LANGSMITH_API_KEY` and
|
||||
`LANGSMITH_ENDPOINT` from the coordinator's environment. Never pass keys in CLI
|
||||
arguments, commit them, or bake them into the snapshot.
|
||||
|
||||
## Lifecycle, recovery and outputs
|
||||
|
||||
- Bounded thread-pool concurrency; each worker has its own SDK client.
|
||||
- Readiness timeout 300 seconds, benchmark subprocess timeout from the plan,
|
||||
and remote execution timeout 120 seconds longer for startup/cleanup.
|
||||
- Completed CSV and metadata are fetched before sandbox deletion. A shard is
|
||||
marked successful only after its sandbox has been deleted.
|
||||
- A shard's `attempt.json` records its sandbox name **before** creation. On a
|
||||
process crash, ambiguous create or failed deletion, inspect that named resource
|
||||
in LangSmith and delete it if it exists. Do not bulk-delete other sandboxes.
|
||||
After verifying it is absent, set this ledger's state to `deleted` to allow
|
||||
retry. An unresolved ledger blocks retry rather than leaking more resources.
|
||||
- Idle stop is 10 minutes and deletion after stop is 10 minutes, providing a
|
||||
fallback for coordinator crashes. This is **not** a strict hard lifetime cap:
|
||||
verify sandbox cleanup after interruption. Active exec/SDK lease behavior can
|
||||
affect idle detection.
|
||||
- A shard failure stops admission of further jobs; already-running jobs finish
|
||||
and clean up. Exceptions shown by workers use only their type, avoiding
|
||||
inadvertent credential exposure. Failed command stdout/stderr are not copied
|
||||
into shared artifacts. Inspect the sandbox before cleanup separately if deep
|
||||
diagnostics are needed; automatic cleanup prioritizes preventing spend.
|
||||
- Rerun the identical plan/output directory to skip successful shards. Failed
|
||||
shards restart from zero; partial remote CSVs are not recovered. Each completed
|
||||
shard is an independent resume unit.
|
||||
- Launch **only one coordinator per output directory**; no cross-process lock is
|
||||
implemented. Do not modify benchmark/worker source during a run. Source hashes
|
||||
and plan equality reject changed-source or changed-budget resume.
|
||||
- Merge requires every expected frequency/profile/length exactly once. Missing,
|
||||
duplicate, foreign and nonfinite rows fail validation. `combined.csv` and SVG
|
||||
plots are emitted only for a complete plan. Per-shard CSV/metadata stay available.
|
||||
|
||||
## Interpretation and verification limits
|
||||
|
||||
See [measurement semantics](delta-results/README.md) before interpreting results.
|
||||
The normalized Pareto compromise is a heuristic, not a universal default. Full
|
||||
state hydration and mock payload compression can dominate the apparent tradeoff.
|
||||
Keep the end-to-end graph as the reference; direct checkpointer seeding is not
|
||||
implemented because equivalence has not been demonstrated. Read sampling remains
|
||||
one read per update to avoid treating assumed snapshot phase as measured truth.
|
||||
|
||||
The coordinator lifecycle and failure paths are locally tested with a fake SDK;
|
||||
the benchmark itself has run against local Postgres. A real remote pilot still
|
||||
needs a prepared snapshot and an agreed concurrency/runtime budget. Do not treat
|
||||
mocked orchestration tests as evidence that a remote fleet has run successfully.
|
||||
@@ -0,0 +1,124 @@
|
||||
# DeltaChannel snapshot-frequency sweep
|
||||
|
||||
## Status and recommendation
|
||||
|
||||
The sequential Postgres grid was stopped in favor of the distributed coordinator.
|
||||
`postgres.csv` contains four completed prefixes from frequency 25 / mock700, not a
|
||||
complete grid. No default change is justified by these partial measurements alone.
|
||||
Until the long-thread grid finishes, retain existing defaults rather than selecting a
|
||||
frequency from short threads. Frequency 300 remains a candidate for both LangGraph
|
||||
and deepagents based on the supplied fleet evidence, not a conclusion of this run.
|
||||
The capped grid does not benchmark LangGraph's current 1000 setting.
|
||||
|
||||
`pilot.csv` contains the initial 25/300/500-frequency, 50/200-update exploratory run.
|
||||
Its first read includes initialization overhead; do not use that maximum as a steady
|
||||
state result. `scaling-pilot.csv` contains 1000-update runs at frequency 300 with
|
||||
warmup. These are explicitly partial grids. `postgres.csv` contains completed points
|
||||
from the requested full grid as it progresses; missing rows are not measurements.
|
||||
|
||||
## Reproduce
|
||||
|
||||
From `libs/langgraph`, sync declared dependencies with `uv sync --frozen --group dev`.
|
||||
Use a **dedicated disposable Postgres database**, not a production database. For
|
||||
example, start `postgres:14.7` with a loopback-only published port and configure
|
||||
`LANGGRAPH_BENCH_POSTGRES_URI` in the environment. The benchmark creates checkpointer
|
||||
tables and deletes only its UUID-scoped threads when each scenario finishes.
|
||||
|
||||
```bash
|
||||
uv run python tests/test_delta_channel_benchmark.py --snapshot-sweep \
|
||||
--backend postgres --output bench/delta-results/postgres.csv
|
||||
uv run python tests/test_delta_channel_benchmark.py --snapshot-sweep \
|
||||
--backend memory --output bench/delta-results/memory.csv
|
||||
```
|
||||
|
||||
Default grid: frequencies 25, 50, ..., 500; lengths 50, 200, 1000, 5000,
|
||||
10000; profiles `mock700` and `tool_calls`. Select a smoke grid with
|
||||
`--frequencies 25 300 500 --lengths 50 200`. The original multi-channel benchmark
|
||||
remains available by omitting `--snapshot-sweep`.
|
||||
|
||||
Every frequency/profile builds one growing thread; shorter lengths are prefixes.
|
||||
Thus each backend performs 20 × 2 × 10000 = 400000 graph invocations and the
|
||||
same number of measured reads, plus warmups and separate heap measurements.
|
||||
There are 200 output rows per complete backend grid. Resume with identical
|
||||
arguments, source, revision and environment metadata: completed rows are skipped.
|
||||
An interrupted scenario is rebuilt from its beginning up to remaining lengths;
|
||||
resuming does not restore a partially written graph. Abrupt process termination
|
||||
may leave the current thread in the disposable database.
|
||||
|
||||
## Workload and measurement semantics
|
||||
|
||||
- Each invocation runs a deterministic `GenericFakeChatModel` script. There are
|
||||
**no provider requests**, sleeps or stochastic outputs. Message and tool-call
|
||||
IDs derive from the update index. Checkpoint IDs and timing vary between runs.
|
||||
- `mock700`: 350 ASCII bytes of human content plus 350 of assistant content.
|
||||
This is **700 content bytes**, not 700 serialized bytes: message envelopes,
|
||||
checkpoint metadata and a file update add overhead.
|
||||
- `tool_calls`: human request, scripted assistant `read_file` call, approximately
|
||||
3.7 KB tool result, then an approximately 600-byte scripted response. It is a
|
||||
synthetic deepagents-style workload, not captured production traffic.
|
||||
- Both `messages` and `files` receive one update in the same node per invocation.
|
||||
Files use deterministic content lines and timestamps, adding a new report path
|
||||
each time. This deliberately models growing file history, not overwriting a
|
||||
fixed set of paths. The reducer merges batches of file dictionaries.
|
||||
- An update is **one graph invocation**, not one individual message. Each
|
||||
invocation has input/start/node checkpoint supersteps; only the node writes
|
||||
the two delta channels. The 5000-superstep bound is retained and recorded via
|
||||
`LANGGRAPH_DELTA_MAX_SUPERSTEPS_SINCE_SNAPSHOT`; the grid does not vary it.
|
||||
- Read statistics describe one warm-cache `get_state` after **every** invocation
|
||||
from update 1 through the reported prefix. They are not repeated final-state
|
||||
reads or a fixed-thread-length phase distribution. This captures pre-snapshot
|
||||
phases as well as snapshots; the maximum is the **observed** slowest read and
|
||||
may reflect system noise, not necessarily the deepest replay chain. Its update
|
||||
and checkpoint ID are recorded. The measured checkpoint is later deleted.
|
||||
- Write statistics time end-to-end `graph.invoke`, including hydration, scripted
|
||||
model CPU work and checkpoint writes; they are **not saver-only SQL latency**.
|
||||
- Heap peak comes from a separate traced read of the observed slowest checkpoint.
|
||||
`tracemalloc` is off for all latency samples. This is Python heap, not RSS or
|
||||
database memory. Reads and writes are sequential, with no concurrent clients.
|
||||
- Memory storage counts serialized checkpoint, metadata, blob and pending-write
|
||||
payload bytes, not Python-object/key overhead. Postgres storage is the sum of
|
||||
`pg_column_size(row)` across checkpoints, blobs and writes scoped to the thread.
|
||||
It is an in-row/TOAST-sensitive estimate, **not total relation/index/WAL/disk
|
||||
usage**, and is not directly interchangeable with the memory payload count.
|
||||
Repeated mock text is highly compressible, especially in Postgres: absolute
|
||||
storage numbers should not be compared with production chat without matching
|
||||
entropy, serialization and storage accounting.
|
||||
- Metadata records interpreter, platform, git revision, script SHA-256, grid and
|
||||
sampling semantics. Initial pilot artifacts predate the final script hash and
|
||||
warmup additions; they are preserved as exploratory evidence, not pooled with
|
||||
the full-grid results. Timings need repeat runs to assess variance.
|
||||
|
||||
## Charts and selecting a frequency
|
||||
|
||||
Each CSV produces an SVG per backend/profile/length. The left panel plots storage,
|
||||
p50 and maximum read latency against frequency, independently divided by each
|
||||
metric's maximum (0–1). These normalized curves **do not have shared units**;
|
||||
their crossing is not a physical optimum. The right panel plots storage bytes
|
||||
against actual p50/max milliseconds.
|
||||
|
||||
A marked compromise point is selected separately for p50 and max: discard
|
||||
storage/latency-dominated points, min-max normalize each objective over that group,
|
||||
and choose the nondominated point nearest the ideal origin in Euclidean distance.
|
||||
This is an equal-weight heuristic, not a universal knee or recommended default.
|
||||
Changing the candidate range, latency statistic or storage/latency weighting can
|
||||
change it. Partial groups display their actual number of frequencies and must not
|
||||
be mistaken for completed sweeps. Frequency points above a short thread's update
|
||||
count can have identical snapshot behavior, making any apparent short-thread
|
||||
optimum particularly noisy.
|
||||
|
||||
For a final recommendation, compare the compromise points across all five thread
|
||||
lengths and both workloads, apply an explicit latency budget, inspect tail noise,
|
||||
and repeat promising frequencies. Keep separate choices for growing message and
|
||||
file workloads if their access/update patterns differ. This suite currently varies
|
||||
both cadences together and cannot identify their independent optimum.
|
||||
|
||||
## Fleet sanity check
|
||||
|
||||
The supplied 10000-turn fleet simulation reports frequency 50: 2.9 GB / 44 ms;
|
||||
300: 488 MB / 244 ms; 500: 307 MB / 404 ms; 1000: 164 MB / 804 ms. Its delta DB time
|
||||
was about 35 ms and about 1% of fleet DB time. These are externally supplied
|
||||
context, **not measurements reproduced here**. Our local Postgres, full graph
|
||||
hydration, files, compressible scripted content and row-size estimate differ.
|
||||
Do not calibrate the results to force a match or equate local end-to-end latency
|
||||
with fleet database time. Storage reduction with less frequent snapshots is a
|
||||
useful directional sanity check, not evidence that absolute fleet values match.
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / mock700 / 200 updates (n=3 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00 320.53,86.47 510.00,86.57" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,161.64 320.53,123.69 510.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00 320.53,293.43 510.00,289.76" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 25 to 500; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="1060.00,327.56 616.92,324.92 610.00,321.87" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="616.9150288807352" cy="324.91598843036314" r="5" fill="#176baf"/><text x="622.9150288807352" y="316.91598843036314" font-family="sans-serif" font-size="11">f=300</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="1060.00,80.00 616.92,293.43 610.00,289.76" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="616.9150288807352" cy="293.42895568807285" r="5" fill="#c14b14"/><text x="622.9150288807352" y="285.42895568807285" font-family="sans-serif" font-size="11">f=300</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 1062349 to 1089876; y: 0 to 197.73 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 2.1 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / mock700 / 50 updates (n=3 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00 320.53,82.82 510.00,82.87" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,153.18 320.53,134.27 510.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00 320.53,325.79 510.00,329.51" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 25 to 500; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="1060.00,335.24 617.01,334.76 610.00,333.38" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="617.005076142132" cy="334.7610466522907" r="5" fill="#176baf"/><text x="623.005076142132" y="326.7610466522907" font-family="sans-serif" font-size="11">f=300</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="1060.00,80.00 617.01,325.79 610.00,329.51" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="329.50511620062144" r="5" fill="#c14b14"/><text x="616.0" y="321.50511620062144" font-family="sans-serif" font-size="11">f=500</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 265018 to 267973; y: 0 to 197.73 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 2.1 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / tool_calls / 200 updates (n=3 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00 320.53,176.58 510.00,176.63" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,153.76 320.53,80.00 510.00,118.12" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00 320.53,171.19 510.00,179.72" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 25 to 500; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="1060.00,288.02 610.24,267.43 610.00,278.07" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="610.0" cy="278.0659453883102" r="5" fill="#176baf"/><text x="616.0" y="270.0659453883102" font-family="sans-serif" font-size="11">f=500</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="1060.00,80.00 610.24,171.19 610.00,179.72" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="179.717130949707" r="5" fill="#c14b14"/><text x="616.0" y="171.717130949707" font-family="sans-serif" font-size="11">f=500</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 1008416 to 1604911; y: 0 to 77.92 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 2.1 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / tool_calls / 50 updates (n=3 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00 320.53,123.32 510.00,123.41" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,80.00 320.53,92.09 510.00,96.96" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00 320.53,124.55 510.00,172.29" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 25 to 500; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="1060.00,240.59 610.96,245.21 610.00,247.07" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="610.0" cy="247.07101749792133" r="5" fill="#176baf"/><text x="616.0" y="239.07101749792133" font-family="sans-serif" font-size="11">f=500</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="1060.00,80.00 610.96,124.55 610.00,172.29" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="172.28887735058402" r="5" fill="#c14b14"/><text x="616.0" y="164.28887735058402" font-family="sans-serif" font-size="11">f=500</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 251530 to 301946; y: 0 to 17.90 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 2.1 KiB |
@@ -0,0 +1,13 @@
|
||||
backend,profile,frequency,updates,read_samples,read_p50_ms,read_p95_ms,read_max_ms,slowest_update,slowest_checkpoint_id,write_p50_ms,write_p95_ms,write_max_ms,write_total_s,storage_bytes,slowest_read_peak_heap_bytes
|
||||
postgres,mock700,25,50,50,3.6180599999511287,6.390288000034161,197.7286589999494,1,1f1cf166-9dd2-6d00-8001-fc71abcfb90a,10.653302000037002,17.0728529999451,30.98109600000498,0.5811482070006377,267973,39327
|
||||
postgres,mock700,25,200,200,9.459105500070564,18.783542000051057,197.7286589999494,1,1f1cf166-9dd2-6d00-8001-fc71abcfb90a,17.477463499972146,30.98109600000498,168.95307799995862,3.8713906139998926,1089876,35220
|
||||
postgres,mock700,300,50,50,3.9841970000225047,6.104986000082135,10.80296499992528,44,1f1cf166-e019-6c30-8082-21934933f621,11.811886500026958,15.252276999945025,28.05887800002438,0.6044334389998767,265064,519513
|
||||
postgres,mock700,300,200,200,11.47131300001547,20.832925000036084,35.41703900009452,185,1f1cf167-0ea3-6a4c-8229-79b49eb7c9fe,19.905765000032716,28.92650700005106,37.699279999969804,3.95925508099981,1062772,2113173
|
||||
postgres,mock700,500,50,50,5.035277500041957,7.819561000019348,7.981305000043903,50,1f1cf167-1fc9-6f84-8094-7f8f188214b6,12.75971849997859,15.276644999971722,16.033585999934985,0.6406766549997656,265018,584506
|
||||
postgres,mock700,500,200,200,13.788552499988782,24.058938000052876,38.209428999948614,185,1f1cf167-5380-6b9e-8229-c84900f5a6db,21.95349900000565,32.21689399993011,39.65822100008154,4.354335682999022,1062349,2110890
|
||||
postgres,tool_calls,25,50,50,6.844815500016921,10.651568999946903,17.9017149999936,50,1f1cf167-6761-6981-8094-f53aeefe50f8,14.597708000053444,19.39097000001766,25.841881000019384,0.7417658820006636,301946,1922124
|
||||
postgres,tool_calls,25,200,200,15.580024999962916,28.78641699999207,77.9230529999495,180,1f1cf167-a207-6b8c-821a-1a5a322fa361,24.694384999975227,42.27366899999652,59.241608999968776,5.190335378001123,1604911,6679384
|
||||
postgres,tool_calls,300,50,50,6.526570999994874,13.39240200002223,14.834163999921657,50,1f1cf167-bca7-6808-8094-f7b7aea71f3d,14.862395000022843,21.3141539999242,43.96138599997812,0.808315155999594,251638,1910246
|
||||
postgres,tool_calls,300,200,200,21.750415999974848,33.799963999967986,50.59318900009657,86,1f1cf167-ce64-6010-8100-78d863dad859,31.301175500004774,48.37651699995149,165.45825800005787,6.2137166139996225,1008731,3276689
|
||||
postgres,tool_calls,500,50,50,6.39841599996771,10.550998999974581,11.547371999995448,49,1f1cf168-2162-68cd-8091-9494a54b4fdd,14.285579000045345,19.514858999968965,23.103371000047446,0.704547747000106,251530,1872876
|
||||
postgres,tool_calls,500,200,200,18.561887000032584,33.225987000037094,48.03742499996133,200,1f1cf168-70e7-6f69-8256-b1395c31fdf1,26.90513599998212,43.38257900008102,53.06229500001791,5.458913677999817,1008416,7571456
|
||||
|
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"python": "3.14.3",
|
||||
"platform": "Linux-6.1.166-x86_64-with-glibc2.43",
|
||||
"revision": "eb066bd31a2bf01fc0cd58256df4800a3695c233",
|
||||
"backend": "postgres",
|
||||
"profiles": [
|
||||
"mock700",
|
||||
"tool_calls"
|
||||
],
|
||||
"frequencies": [
|
||||
25,
|
||||
300,
|
||||
500
|
||||
],
|
||||
"lengths": [
|
||||
50,
|
||||
200
|
||||
],
|
||||
"script": "deterministic, no provider; IDs follow update index",
|
||||
"read_sampling": "one get_state after every invocation, prefix distribution",
|
||||
"write_sampling": "end-to-end graph.invoke including hydrate, scripted model and checkpoint I/O",
|
||||
"storage": "postgres pg_column_size row estimate (not relation disk bytes); memory serialized payloads across all stores",
|
||||
"max_supersteps_since_snapshot": "5000"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / mock700 / 1000 updates (n=1 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 25 to 25; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="610.00,270.42" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="610.0" cy="270.41705212026505" r="5" fill="#176baf"/><text x="616.0" y="262.41705212026505" font-family="sans-serif" font-size="11">f=25</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="610.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="80.0" r="5" fill="#c14b14"/><text x="616.0" y="72.0" font-family="sans-serif" font-size="11">f=25</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 5864394 to 5864394; y: 0 to 88.08 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / mock700 / 200 updates (n=1 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 25 to 25; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="610.00,225.40" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="610.0" cy="225.39966722149467" r="5" fill="#176baf"/><text x="616.0" y="217.39966722149467" font-family="sans-serif" font-size="11">f=25</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="610.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="80.0" r="5" fill="#c14b14"/><text x="616.0" y="72.0" font-family="sans-serif" font-size="11">f=25</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 1089890 to 1089890; y: 0 to 16.54 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / mock700 / 50 updates (n=1 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 25 to 25; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="610.00,184.65" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="610.0" cy="184.6472608451244" r="5" fill="#176baf"/><text x="616.0" y="176.6472608451244" font-family="sans-serif" font-size="11">f=25</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="610.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="80.0" r="5" fill="#c14b14"/><text x="616.0" y="72.0" font-family="sans-serif" font-size="11">f=25</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 267967 to 267967; y: 0 to 5.21 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / mock700 / 5000 updates (n=1 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 25 to 25; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="610.00,274.30" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="610.0" cy="274.2962318424321" r="5" fill="#176baf"/><text x="616.0" y="266.2962318424321" font-family="sans-serif" font-size="11">f=25</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="610.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="80.0" r="5" fill="#c14b14"/><text x="616.0" y="72.0" font-family="sans-serif" font-size="11">f=25</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 39599985 to 39599985; y: 0 to 302.30 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1,5 @@
|
||||
backend,profile,frequency,updates,read_samples,read_p50_ms,read_p95_ms,read_max_ms,slowest_update,slowest_checkpoint_id,write_p50_ms,write_p95_ms,write_max_ms,write_total_s,storage_bytes,slowest_read_peak_heap_bytes
|
||||
postgres,mock700,25,50,50,3.110554999921078,4.785336999930223,5.2058580001812516,46,1f1cf189-c1d7-643f-8088-2d4ca5a5b740,10.314074000007167,12.664818999837735,27.913941999941017,0.53071240700001,267967,483221
|
||||
postgres,mock700,25,200,200,7.288281000001007,13.18430499986789,16.53531900001326,190,1f1cf189-e494-60d8-8238-d44b3e1f2bf0,15.244594500018138,22.50665300016408,37.84130200006075,3.0723227170001337,1089890,1720217
|
||||
postgres,mock700,25,1000,1000,23.5726189999923,34.56026000003476,88.08021400000143,785,1f1cf18b-25f4-6d27-8931-fc21b750ad8c,32.47065949994976,51.13759799996842,134.2724809999254,31.385343082000187,5864394,5889532
|
||||
postgres,mock700,25,5000,5000,76.39274549990205,140.11053600006562,302.2979410000062,3412,1f1cf199-7e0d-6560-a7fa-9687f00ad7cc,90.85022650003793,170.46306499969432,414.5206420002978,459.44821525399425,39599985,22671874
|
||||
|
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"python": "3.14.3",
|
||||
"platform": "Linux-6.1.166-x86_64-with-glibc2.43",
|
||||
"script_sha256": "3a3cb80dde575f1f008c9078e285cc2e8f8a3bb8648da4c3817cb33f9e9a4aed",
|
||||
"revision": "eb066bd31a2bf01fc0cd58256df4800a3695c233",
|
||||
"backend": "postgres",
|
||||
"profiles": [
|
||||
"mock700",
|
||||
"tool_calls"
|
||||
],
|
||||
"frequencies": [
|
||||
25,
|
||||
50,
|
||||
75,
|
||||
100,
|
||||
125,
|
||||
150,
|
||||
175,
|
||||
200,
|
||||
225,
|
||||
250,
|
||||
275,
|
||||
300,
|
||||
325,
|
||||
350,
|
||||
375,
|
||||
400,
|
||||
425,
|
||||
450,
|
||||
475,
|
||||
500
|
||||
],
|
||||
"lengths": [
|
||||
50,
|
||||
200,
|
||||
1000,
|
||||
5000,
|
||||
10000
|
||||
],
|
||||
"script": "deterministic, no provider; IDs follow update index",
|
||||
"read_sampling": "one get_state after every invocation, prefix distribution",
|
||||
"write_sampling": "end-to-end graph.invoke including hydrate, scripted model and checkpoint I/O",
|
||||
"storage": "postgres pg_column_size row estimate (not relation disk bytes); memory serialized payloads across all stores",
|
||||
"max_supersteps_since_snapshot": "5000"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / mock700 / 1000 updates (n=1 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 300 to 300; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="610.00,278.82" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="610.0" cy="278.815545790787" r="5" fill="#176baf"/><text x="616.0" y="270.815545790787" font-family="sans-serif" font-size="11">f=300</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="610.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="80.0" r="5" fill="#c14b14"/><text x="616.0" y="72.0" font-family="sans-serif" font-size="11">f=300</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 5365127 to 5365127; y: 0 to 123.55 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/><text x="20" y="25" font-family="sans-serif" font-size="16">postgres / tool_calls / 1000 updates (n=1 frequencies)</text><path d="M 60 60 V 340 H 510" fill="none" stroke="black"/><text x="60" y="50" font-family="sans-serif" font-size="12">Frequency vs independently normalized metrics</text><polyline points="60.00,80.00" fill="none" stroke="#176baf" stroke-width="2"/><text x="60" y="365" fill="#176baf" font-family="sans-serif" font-size="11">storage_bytes</text><polyline points="60.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><text x="60" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="60.00,80.00" fill="none" stroke="#6a378b" stroke-width="2"/><text x="60" y="395" fill="#6a378b" font-family="sans-serif" font-size="11">read_max_ms</text><text x="60" y="355" font-family="sans-serif" font-size="10">frequency: 300 to 300; y: 0 to 1.00 normalized</text><path d="M 610 60 V 340 H 1060" fill="none" stroke="black"/><text x="610" y="50" font-family="sans-serif" font-size="12">Storage vs latency (Pareto tradeoff)</text><polyline points="610.00,281.99" fill="none" stroke="#176baf" stroke-width="2"/><circle cx="610.0" cy="281.9919586818079" r="5" fill="#176baf"/><text x="616.0" y="273.9919586818079" font-family="sans-serif" font-size="11">f=300</text><text x="610" y="365" fill="#176baf" font-family="sans-serif" font-size="11">read_p50_ms</text><polyline points="610.00,80.00" fill="none" stroke="#c14b14" stroke-width="2"/><circle cx="610.0" cy="80.0" r="5" fill="#c14b14"/><text x="616.0" y="72.0" font-family="sans-serif" font-size="11">f=300</text><text x="610" y="380" fill="#c14b14" font-family="sans-serif" font-size="11">read_max_ms</text><text x="610" y="355" font-family="sans-serif" font-size="10">storage_bytes: 6230423 to 6230423; y: 0 to 261.54 ms</text></svg>
|
||||
|
After Width: | Height: | Size: 1.9 KiB |
@@ -0,0 +1,3 @@
|
||||
backend,profile,frequency,updates,read_samples,read_p50_ms,read_p95_ms,read_max_ms,slowest_update,slowest_checkpoint_id,write_p50_ms,write_p95_ms,write_max_ms,write_total_s,storage_bytes,slowest_read_peak_heap_bytes
|
||||
postgres,mock700,300,1000,1000,29.075356000021202,49.26721499998621,123.55413900002077,339,1f1cf16e-fccf-68b6-83f7-54e1839f7866,38.3128815000191,57.815339000057975,117.20152399993822,37.95603520900272,5365127,3107507
|
||||
postgres,tool_calls,300,1000,1000,58.35261599997921,106.53731800005062,261.5444310000612,900,1f1cf175-2376-6bf5-8a8a-ce7f009ad20f,70.02236149997998,122.92945799993049,376.99114200006534,71.72656674900168,6230423,34206365
|
||||
|
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"python": "3.14.3",
|
||||
"platform": "Linux-6.1.166-x86_64-with-glibc2.43",
|
||||
"script_sha256": "3a3cb80dde575f1f008c9078e285cc2e8f8a3bb8648da4c3817cb33f9e9a4aed",
|
||||
"revision": "eb066bd31a2bf01fc0cd58256df4800a3695c233",
|
||||
"backend": "postgres",
|
||||
"profiles": [
|
||||
"mock700",
|
||||
"tool_calls"
|
||||
],
|
||||
"frequencies": [
|
||||
300
|
||||
],
|
||||
"lengths": [
|
||||
1000
|
||||
],
|
||||
"script": "deterministic, no provider; IDs follow update index",
|
||||
"read_sampling": "one get_state after every invocation, prefix distribution",
|
||||
"write_sampling": "end-to-end graph.invoke including hydrate, scripted model and checkpoint I/O",
|
||||
"storage": "postgres pg_column_size row estimate (not relation disk bytes); memory serialized payloads across all stores",
|
||||
"max_supersteps_since_snapshot": "5000"
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
FROM ghcr.io/astral-sh/uv:0.13.4 AS uv
|
||||
FROM postgres:14.7
|
||||
COPY --from=uv /uv /usr/local/bin/uv
|
||||
ENV UV_PYTHON_INSTALL_DIR=/opt/python UV_PYTHON_PREFERENCE=only-managed
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
RUN uv python install 3.14.3
|
||||
ARG LANGGRAPH_REVISION
|
||||
RUN test "${#LANGGRAPH_REVISION}" -eq 40 \
|
||||
&& git clone https://github.com/langchain-ai/langgraph.git /workspace/langgraph \
|
||||
&& git -C /workspace/langgraph checkout --detach "$LANGGRAPH_REVISION" \
|
||||
&& test "$(git -C /workspace/langgraph rev-parse HEAD)" = "$LANGGRAPH_REVISION"
|
||||
WORKDIR /workspace/langgraph/libs/langgraph
|
||||
RUN uv sync --frozen --group dev --python 3.14.3 \
|
||||
&& uv cache clean \
|
||||
&& .venv/bin/python -c 'from langsmith.sandbox import SandboxClient; from langgraph.checkpoint.postgres import PostgresSaver' \
|
||||
&& command -v initdb && command -v pg_ctl
|
||||
ENTRYPOINT []
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Plan, execute and merge isolated LangSmith sandbox benchmark shards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import itertools
|
||||
import json
|
||||
import shlex
|
||||
import threading
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from langsmith.sandbox import SandboxClient
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BENCHMARK = ROOT / "tests/test_delta_channel_benchmark.py"
|
||||
REMOTE = "/workspace/langgraph/libs/langgraph"
|
||||
PROFILES = ("mock700", "tool_calls")
|
||||
FREQUENCIES = tuple(range(25, 501, 25))
|
||||
LENGTHS = (50, 200, 1000, 5000, 10000)
|
||||
|
||||
|
||||
def _atomic_json(path: Path, value: Any) -> None:
|
||||
temp = path.with_suffix(".tmp")
|
||||
temp.write_text(json.dumps(value, indent=2) + "\n")
|
||||
temp.replace(path)
|
||||
|
||||
|
||||
def make_plan(args: argparse.Namespace) -> dict:
|
||||
"""Bind every shard to source, snapshot, resource and runtime settings."""
|
||||
return {
|
||||
"schema": 1,
|
||||
"snapshot_id": args.snapshot_id,
|
||||
"concurrency": args.concurrency,
|
||||
"timeout_seconds": args.timeout,
|
||||
"vcpus": args.vcpus,
|
||||
"mem_bytes": args.memory_gib * 1024**3,
|
||||
"fs_capacity_bytes": args.disk_gib * 1024**3,
|
||||
"source_sha256": hashlib.sha256(BENCHMARK.read_bytes()).hexdigest(),
|
||||
"worker_sha256": hashlib.sha256(
|
||||
Path(__file__).with_name("delta_shard.py").read_bytes()
|
||||
).hexdigest(),
|
||||
"lengths": sorted(set(args.lengths)),
|
||||
"shards": [
|
||||
{"key": f"{profile}-{freq}", "profile": profile, "frequency": freq}
|
||||
for profile, freq in itertools.product(args.profiles, args.frequencies)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def validate_csv(data: bytes, shard: dict, lengths: list[int]) -> list[dict]:
|
||||
"""Reject missing, duplicate, foreign or malformed measurement points."""
|
||||
rows = list(csv.DictReader(io.StringIO(data.decode())))
|
||||
actual = []
|
||||
for row in rows:
|
||||
if (
|
||||
row["backend"] != "postgres"
|
||||
or row["profile"] != shard["profile"]
|
||||
or int(row["frequency"]) != shard["frequency"]
|
||||
):
|
||||
raise ValueError("Foreign shard row")
|
||||
actual.append(int(row["updates"]))
|
||||
for metric in ("read_p50_ms", "read_p95_ms", "read_max_ms", "storage_bytes"):
|
||||
value = float(row[metric])
|
||||
if not 0 <= value < float("inf"):
|
||||
raise ValueError("Nonfinite or negative metric")
|
||||
if sorted(actual) != sorted(lengths):
|
||||
raise ValueError("Incomplete or duplicate shard lengths")
|
||||
return rows
|
||||
|
||||
|
||||
def run_shard(
|
||||
plan: dict, shard: dict, directory: Path, client_factory=SandboxClient
|
||||
) -> None:
|
||||
"""Use a disposable sandbox and save validated outputs before deleting it."""
|
||||
target = directory / shard["key"]
|
||||
target.mkdir(exist_ok=True)
|
||||
success = target / "success.json"
|
||||
if success.exists():
|
||||
validate_csv((target / "results.csv").read_bytes(), shard, plan["lengths"])
|
||||
if json.loads(success.read_text())["source_sha256"] != plan["source_sha256"]:
|
||||
raise ValueError("Completed shard source differs")
|
||||
return
|
||||
ledger = target / "attempt.json"
|
||||
if ledger.exists() and json.loads(ledger.read_text())["state"] != "deleted":
|
||||
raise RuntimeError(
|
||||
f"Inspect and clean up the previous sandbox in {ledger} before retrying"
|
||||
)
|
||||
sandbox_name = f"delta-bench-{uuid.uuid4().hex}"
|
||||
_atomic_json(ledger, {"sandbox_name": sandbox_name, "state": "creating"})
|
||||
with client_factory() as client:
|
||||
sandbox = None
|
||||
completed = False
|
||||
try:
|
||||
sandbox = client.create_sandbox(
|
||||
snapshot_id=plan["snapshot_id"],
|
||||
name=sandbox_name,
|
||||
timeout=300,
|
||||
idle_ttl_seconds=600,
|
||||
delete_after_stop_seconds=600,
|
||||
vcpus=plan["vcpus"],
|
||||
mem_bytes=plan["mem_bytes"],
|
||||
fs_capacity_bytes=plan["fs_capacity_bytes"],
|
||||
)
|
||||
_atomic_json(ledger, {"sandbox_name": sandbox_name, "state": "running"})
|
||||
sandbox.write(
|
||||
f"{REMOTE}/tests/test_delta_channel_benchmark.py",
|
||||
BENCHMARK.read_bytes(),
|
||||
)
|
||||
sandbox.write(
|
||||
"/tmp/delta_shard.py",
|
||||
Path(__file__).with_name("delta_shard.py").read_bytes(),
|
||||
)
|
||||
sandbox.write(
|
||||
"/tmp/delta-job.json",
|
||||
json.dumps(
|
||||
{
|
||||
**shard,
|
||||
"lengths": plan["lengths"],
|
||||
"timeout_seconds": plan["timeout_seconds"],
|
||||
}
|
||||
),
|
||||
)
|
||||
result = sandbox.run(
|
||||
shlex.join([f"{REMOTE}/.venv/bin/python", "/tmp/delta_shard.py"]),
|
||||
timeout=plan["timeout_seconds"] + 120,
|
||||
cwd=REMOTE,
|
||||
kill_on_disconnect=True,
|
||||
)
|
||||
if result.exit_code != 0:
|
||||
raise RuntimeError(
|
||||
"Shard command failed; sandbox logs withheld from shared output"
|
||||
)
|
||||
data = sandbox.read("/tmp/delta-results.csv")
|
||||
metadata = json.loads(sandbox.read("/tmp/delta-results.metadata.json"))
|
||||
if metadata["script_sha256"] != plan["source_sha256"]:
|
||||
raise ValueError("Remote benchmark source differs")
|
||||
validate_csv(data, shard, plan["lengths"])
|
||||
(target / "results.csv.tmp").write_bytes(data)
|
||||
(target / "results.csv.tmp").replace(target / "results.csv")
|
||||
_atomic_json(target / "metadata.json", metadata)
|
||||
completed = True
|
||||
finally:
|
||||
try:
|
||||
client.delete_sandbox(sandbox_name)
|
||||
except Exception:
|
||||
_atomic_json(
|
||||
ledger, {"sandbox_name": sandbox_name, "state": "cleanup_required"}
|
||||
)
|
||||
raise RuntimeError(f"Sandbox cleanup required; see {ledger}") from None
|
||||
_atomic_json(ledger, {"sandbox_name": sandbox_name, "state": "deleted"})
|
||||
if completed:
|
||||
_atomic_json(
|
||||
success,
|
||||
{"source_sha256": plan["source_sha256"], "sandbox_name": sandbox_name},
|
||||
)
|
||||
|
||||
|
||||
def merge(plan: dict, directory: Path) -> Path:
|
||||
"""Require the entire planned grid before publishing a combined CSV."""
|
||||
rows = []
|
||||
for shard in plan["shards"]:
|
||||
target = directory / shard["key"]
|
||||
if not (target / "success.json").exists():
|
||||
raise ValueError(f"Shard incomplete: {shard['key']}")
|
||||
rows.extend(
|
||||
validate_csv((target / "results.csv").read_bytes(), shard, plan["lengths"])
|
||||
)
|
||||
output = directory / "combined.csv"
|
||||
temp = output.with_suffix(".tmp")
|
||||
with temp.open("w", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=list(rows[0]), lineterminator="\n")
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
temp.replace(output)
|
||||
spec = importlib.util.spec_from_file_location("delta_benchmark", BENCHMARK)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
module._plot_sweep(output)
|
||||
return output
|
||||
|
||||
|
||||
def execute(plan: dict, directory: Path, client_factory=SandboxClient) -> None:
|
||||
"""Bound active sandboxes and stop admitting shards after a failure."""
|
||||
failed = threading.Event()
|
||||
|
||||
def run(shard: dict) -> None:
|
||||
if failed.is_set():
|
||||
return
|
||||
try:
|
||||
run_shard(plan, shard, directory, client_factory)
|
||||
except Exception as exc:
|
||||
failed.set()
|
||||
print(f"{shard['key']}: {type(exc).__name__}; see shard ledger", flush=True)
|
||||
raise
|
||||
|
||||
with ThreadPoolExecutor(max_workers=plan["concurrency"]) as pool:
|
||||
list(pool.map(run, plan["shards"]))
|
||||
merge(plan, directory)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--snapshot-id", required=True)
|
||||
parser.add_argument("--concurrency", type=int, required=True)
|
||||
parser.add_argument(
|
||||
"--timeout", type=int, required=True, help="Maximum benchmark seconds per shard"
|
||||
)
|
||||
parser.add_argument("--vcpus", type=int, required=True)
|
||||
parser.add_argument("--memory-gib", type=int, required=True)
|
||||
parser.add_argument("--disk-gib", type=int, required=True)
|
||||
parser.add_argument("--frequencies", nargs="+", type=int, default=list(FREQUENCIES))
|
||||
parser.add_argument("--lengths", nargs="+", type=int, default=list(LENGTHS))
|
||||
parser.add_argument(
|
||||
"--profiles", nargs="+", choices=PROFILES, default=list(PROFILES)
|
||||
)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--execute",
|
||||
action="store_true",
|
||||
help="Create billable sandboxes; otherwise only write a plan",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if any(
|
||||
n <= 0
|
||||
for n in [
|
||||
args.concurrency,
|
||||
args.timeout,
|
||||
args.vcpus,
|
||||
args.memory_gib,
|
||||
args.disk_gib,
|
||||
*args.frequencies,
|
||||
*args.lengths,
|
||||
]
|
||||
):
|
||||
parser.error("Resource limits, frequencies and lengths must be positive")
|
||||
if args.concurrency > 40 or args.timeout > 86400:
|
||||
parser.error("Concurrency must be <=40 and shard timeout <=86400 seconds")
|
||||
if len(set(args.frequencies)) != len(args.frequencies) or len(
|
||||
set(args.profiles)
|
||||
) != len(args.profiles):
|
||||
parser.error("Duplicate shard parameters")
|
||||
directory = args.output.resolve()
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
plan = make_plan(args)
|
||||
plan_path = directory / "plan.json"
|
||||
if plan_path.exists() and json.loads(plan_path.read_text()) != plan:
|
||||
parser.error("Existing plan differs; use a new output directory")
|
||||
_atomic_json(plan_path, plan)
|
||||
print(
|
||||
f"{len(plan['shards'])} shards; concurrency {args.concurrency}; upper bound {len(plan['shards']) * (args.timeout + 420) / 3600:.2f} sandbox-hours excluding deletion; execute={args.execute}"
|
||||
)
|
||||
if args.execute:
|
||||
execute(plan, directory)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Execute one scripted benchmark with a sandbox-local disposable Postgres."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path("/workspace/langgraph/libs/langgraph")
|
||||
PGDATA = Path("/tmp/delta-bench-pg")
|
||||
|
||||
|
||||
def checked(command: list[str], **kwargs) -> None:
|
||||
subprocess.run(command, check=True, stdout=subprocess.DEVNULL, **kwargs)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
job = json.loads(Path("/tmp/delta-job.json").read_text())
|
||||
prefix = ["runuser", "-u", "postgres", "--"] if os.getuid() == 0 else []
|
||||
for binary in ("initdb", "pg_ctl"):
|
||||
if shutil.which(binary) is None:
|
||||
raise RuntimeError(f"Snapshot must provide {binary} on PATH")
|
||||
PGDATA.mkdir(mode=0o700)
|
||||
if prefix:
|
||||
checked(["chown", "postgres:postgres", str(PGDATA)])
|
||||
checked(
|
||||
[
|
||||
*prefix,
|
||||
"initdb",
|
||||
"-D",
|
||||
str(PGDATA),
|
||||
"--auth-local=trust",
|
||||
"--auth-host=trust",
|
||||
"-U",
|
||||
"postgres",
|
||||
]
|
||||
)
|
||||
started = False
|
||||
try:
|
||||
checked(
|
||||
[
|
||||
*prefix,
|
||||
"pg_ctl",
|
||||
"-D",
|
||||
str(PGDATA),
|
||||
"-l",
|
||||
"/tmp/delta-postgres.log",
|
||||
"-o",
|
||||
"-h 127.0.0.1 -p 5432 -k /tmp",
|
||||
"-w",
|
||||
"start",
|
||||
]
|
||||
)
|
||||
started = True
|
||||
command = [
|
||||
sys.executable,
|
||||
str(ROOT / "tests/test_delta_channel_benchmark.py"),
|
||||
"--snapshot-sweep",
|
||||
"--backend",
|
||||
"postgres",
|
||||
"--frequencies",
|
||||
str(job["frequency"]),
|
||||
"--profiles",
|
||||
job["profile"],
|
||||
"--lengths",
|
||||
*map(str, job["lengths"]),
|
||||
"--output",
|
||||
"/tmp/delta-results.csv",
|
||||
]
|
||||
env = {
|
||||
**os.environ,
|
||||
"LANGGRAPH_BENCH_POSTGRES_URI": "postgresql://postgres@127.0.0.1:5432/postgres?sslmode=disable",
|
||||
"LANGSMITH_TRACING": "false",
|
||||
"LANGCHAIN_TRACING_V2": "false",
|
||||
"NO_COLOR": "1",
|
||||
}
|
||||
subprocess.run(
|
||||
command, cwd=ROOT, env=env, timeout=job["timeout_seconds"], check=True
|
||||
)
|
||||
finally:
|
||||
if started:
|
||||
checked(
|
||||
[*prefix, "pg_ctl", "-D", str(PGDATA), "-m", "immediate", "-w", "stop"]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Build a dedicated LangSmith benchmark snapshot from a minimal Docker context."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from langsmith.sandbox import SandboxClient
|
||||
|
||||
CONTEXT = Path(__file__).resolve().with_name("delta-snapshot")
|
||||
|
||||
|
||||
def snapshot_plan(args: argparse.Namespace) -> dict:
|
||||
"""Validate resource bounds and pin the public repository to a full SHA."""
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", args.revision):
|
||||
raise ValueError("Revision must be a full lowercase 40-character commit SHA")
|
||||
if not re.fullmatch(r"[a-z0-9][a-z0-9-]{0,62}", args.name):
|
||||
raise ValueError(
|
||||
"Name must contain only lowercase letters, numbers and hyphens"
|
||||
)
|
||||
if not (
|
||||
1 <= args.vcpus <= 16
|
||||
and 1 <= args.memory_gib <= 64
|
||||
and 10 <= args.disk_gib <= 100
|
||||
and 60 <= args.timeout <= 7200
|
||||
):
|
||||
raise ValueError(
|
||||
"Build limits: CPU 1–16, memory 1–64 GiB, disk 10–100 GiB, timeout 60–7200 seconds"
|
||||
)
|
||||
if sorted(p.name for p in CONTEXT.iterdir()) != ["Dockerfile"]:
|
||||
raise ValueError(
|
||||
"Build context must contain only Dockerfile; refusing other files"
|
||||
)
|
||||
return {
|
||||
"name": args.name,
|
||||
"revision": args.revision,
|
||||
"dockerfile_sha256": hashlib.sha256(
|
||||
(CONTEXT / "Dockerfile").read_bytes()
|
||||
).hexdigest(),
|
||||
"vcpus": args.vcpus,
|
||||
"mem_bytes": args.memory_gib * 1024**3,
|
||||
"fs_capacity_bytes": args.disk_gib * 1024**3,
|
||||
"timeout": args.timeout,
|
||||
}
|
||||
|
||||
|
||||
def build_snapshot(plan: dict, client_factory=SandboxClient) -> dict:
|
||||
"""Create a billable builder, then return the reusable snapshot receipt."""
|
||||
with client_factory() as client:
|
||||
snapshot = client.create_snapshot_from_dockerfile(
|
||||
name=plan["name"],
|
||||
dockerfile="Dockerfile",
|
||||
context=CONTEXT,
|
||||
fs_capacity_bytes=plan["fs_capacity_bytes"],
|
||||
build_args={"LANGGRAPH_REVISION": plan["revision"]},
|
||||
vcpus=plan["vcpus"],
|
||||
mem_bytes=plan["mem_bytes"],
|
||||
timeout=plan["timeout"],
|
||||
)
|
||||
return {**plan, "snapshot_id": snapshot.id, "status": snapshot.status}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--name", required=True)
|
||||
parser.add_argument(
|
||||
"--revision",
|
||||
default=None,
|
||||
help="Full public langgraph commit SHA; defaults to local HEAD",
|
||||
)
|
||||
parser.add_argument("--vcpus", type=int, default=2)
|
||||
parser.add_argument("--memory-gib", type=int, default=8)
|
||||
parser.add_argument("--disk-gib", type=int, default=30)
|
||||
parser.add_argument("--timeout", type=int, default=1800)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--execute",
|
||||
action="store_true",
|
||||
help="Create a billable builder and persistent snapshot",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error(
|
||||
"Output already exists; use a fresh receipt path to avoid duplicate builds"
|
||||
)
|
||||
if args.revision is None:
|
||||
args.revision = subprocess.check_output(
|
||||
["git", "-C", str(CONTEXT), "rev-parse", "HEAD"], text=True
|
||||
).strip()
|
||||
try:
|
||||
plan = snapshot_plan(args)
|
||||
except ValueError as exc:
|
||||
parser.error(str(exc))
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps({**plan, "state": "planned"}, indent=2) + "\n")
|
||||
print(json.dumps(plan, indent=2))
|
||||
if not args.execute:
|
||||
print(
|
||||
"Dry run: no resources created. Use --execute and a fresh receipt path to build."
|
||||
)
|
||||
return
|
||||
args.output.write_text(json.dumps({**plan, "state": "building"}, indent=2) + "\n")
|
||||
receipt = build_snapshot(plan)
|
||||
args.output.write_text(json.dumps(receipt, indent=2) + "\n")
|
||||
print(f"Snapshot ready: {receipt['snapshot_id']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -20,16 +20,29 @@ Scenarios cover the dimensions where this branch's optimizations matter:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import csv
|
||||
import gc
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import tracemalloc
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langchain_core.language_models.fake_chat_models import GenericFakeChatModel
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
@@ -210,10 +223,10 @@ def _pg_saver(thread_id: str):
|
||||
with saver._cursor() as cur:
|
||||
for tbl in ("checkpoints", "checkpoint_blobs", "checkpoint_writes"):
|
||||
cur.execute(f"DELETE FROM {tbl} WHERE thread_id = %s", (thread_id,))
|
||||
yield saver
|
||||
with saver._cursor() as cur:
|
||||
for tbl in ("checkpoints", "checkpoint_blobs", "checkpoint_writes"):
|
||||
cur.execute(f"DELETE FROM {tbl} WHERE thread_id = %s", (thread_id,))
|
||||
try:
|
||||
yield saver
|
||||
finally:
|
||||
saver.delete_thread(thread_id)
|
||||
|
||||
|
||||
def _checkpointers() -> list[tuple[str, Any]]:
|
||||
@@ -314,8 +327,389 @@ def test_delta_channel_benchmark(capsys: Any) -> None:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_scripted_snapshot_sweep() -> None:
|
||||
saver = MemorySaver()
|
||||
graph = _scripted_graph(2, "tool_calls", saver)
|
||||
config = {"configurable": {"thread_id": "scripted-test"}}
|
||||
for _ in range(3):
|
||||
graph.invoke({}, config)
|
||||
state = graph.get_state(config).values
|
||||
assert len(state["messages"]) == 12
|
||||
assert len(state["files"]) == 3
|
||||
assert state["messages"][1].tool_calls[0]["id"] == "call-0"
|
||||
assert state["messages"][2].tool_call_id == "call-0"
|
||||
assert graph.get_state(config).values == state
|
||||
assert _memory_storage_bytes(saver, "scripted-test") > 0
|
||||
|
||||
|
||||
def test_sweep_metrics_and_cleanup() -> None:
|
||||
rows = list(_measure_sweep(2, [2, 4], "mock700", "memory"))
|
||||
assert [row["read_samples"] for row in rows] == [2, 4]
|
||||
assert rows[1]["storage_bytes"] > rows[0]["storage_bytes"]
|
||||
assert rows[1]["read_max_ms"] >= rows[1]["read_p95_ms"] >= rows[1]["read_p50_ms"]
|
||||
assert 1 <= rows[1]["slowest_update"] <= 4
|
||||
assert _percentile([1, 2, 3, 4], 0.95) == 4
|
||||
points = [
|
||||
{"storage_bytes": 100, "read_max_ms": 1, "frequency": 25},
|
||||
{"storage_bytes": 50, "read_max_ms": 2, "frequency": 50},
|
||||
{"storage_bytes": 10, "read_max_ms": 10, "frequency": 100},
|
||||
]
|
||||
assert _pareto_knee(points, "read_max_ms")["frequency"] == 50
|
||||
|
||||
|
||||
FREQUENCIES = list(range(25, 501, 25))
|
||||
LENGTHS = [50, 200, 1000, 5000, 10000]
|
||||
PROFILES = ["mock700", "tool_calls"]
|
||||
|
||||
|
||||
def _files_delta(state: dict, writes: Sequence[dict]) -> dict:
|
||||
return {**state, **{key: value for write in writes for key, value in write.items()}}
|
||||
|
||||
|
||||
def _script(i: int, profile: str) -> list:
|
||||
"""Return reproducible, ID-stable messages without contacting a provider."""
|
||||
if profile == "mock700":
|
||||
return [
|
||||
HumanMessage(content="h" * 350, id=f"human-{i}"),
|
||||
AIMessage(content="a" * 350, id=f"answer-{i}"),
|
||||
]
|
||||
call_id = f"call-{i}"
|
||||
return [
|
||||
HumanMessage(content=_human_content(i), id=f"human-{i}"),
|
||||
AIMessage(
|
||||
content="I will inspect the source and write a report.",
|
||||
id=f"call-message-{i}",
|
||||
tool_calls=[
|
||||
{"name": "read_file", "args": {"path": f"/src/{i}.py"}, "id": call_id}
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
content=(
|
||||
"def transform(record):\n return record.get('value', 0)\n" * 64
|
||||
),
|
||||
tool_call_id=call_id,
|
||||
id=f"tool-{i}",
|
||||
),
|
||||
AIMessage(
|
||||
content="The implementation is deterministic.\n" * 16, id=f"answer-{i}"
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _scripted_graph(freq: int, profile: str, saver: Any) -> Any:
|
||||
"""Append one scripted conversation and file update per graph invocation."""
|
||||
fields = {
|
||||
"messages": Annotated[
|
||||
list, DeltaChannel(_messages_delta_reducer, snapshot_frequency=freq)
|
||||
],
|
||||
"files": Annotated[dict, DeltaChannel(_files_delta, snapshot_frequency=freq)],
|
||||
}
|
||||
state_cls = TypedDict("ScriptedState", fields)
|
||||
counter = itertools.count()
|
||||
|
||||
def turn(state: dict) -> dict:
|
||||
i = next(counter)
|
||||
script = _script(i, profile)
|
||||
model = GenericFakeChatModel(
|
||||
messages=iter(m for m in script if isinstance(m, AIMessage))
|
||||
)
|
||||
history = list(state.get("messages", []))
|
||||
emitted = []
|
||||
for message in script:
|
||||
actual = (
|
||||
model.invoke(history) if isinstance(message, AIMessage) else message
|
||||
)
|
||||
emitted.append(actual)
|
||||
history.append(actual)
|
||||
content = "report\n" * (16 if profile == "mock700" else 256)
|
||||
return {
|
||||
"messages": emitted,
|
||||
"files": {
|
||||
f"/reports/{i}.txt": {
|
||||
"content": content.splitlines(),
|
||||
"created_at": "2026-01-01T00:00:00Z",
|
||||
"modified_at": "2026-01-01T00:00:00Z",
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
builder = StateGraph(state_cls)
|
||||
builder.add_node("scripted_chat", turn)
|
||||
builder.set_entry_point("scripted_chat")
|
||||
builder.add_edge("scripted_chat", END)
|
||||
return builder.compile(checkpointer=saver)
|
||||
|
||||
|
||||
def _percentile(values: list[float], percentile: float) -> float:
|
||||
return sorted(values)[max(0, math.ceil(len(values) * percentile) - 1)]
|
||||
|
||||
|
||||
def _memory_storage_bytes(saver: MemorySaver, thread_id: str) -> int:
|
||||
"""Count serialized payload bytes across all three memory saver stores."""
|
||||
blobs = sum(
|
||||
len(blob)
|
||||
for (tid, *_), (_, blob) in saver.blobs.items()
|
||||
if tid == thread_id and blob is not None
|
||||
)
|
||||
checkpoints = sum(
|
||||
len(checkpoint[1]) + len(metadata[1])
|
||||
for ns in saver.storage[thread_id].values()
|
||||
for checkpoint, metadata, _ in ns.values()
|
||||
)
|
||||
writes = sum(
|
||||
len(value[2][1])
|
||||
for (tid, *_), batch in saver.writes.items()
|
||||
if tid == thread_id
|
||||
for value in batch.values()
|
||||
)
|
||||
return blobs + checkpoints + writes
|
||||
|
||||
|
||||
def _measure_sweep(freq: int, lengths: list[int], profile: str, backend: str):
|
||||
"""Measure every prefix read, including pre-snapshot phases, without tracing."""
|
||||
thread_id = f"delta-sweep-{uuid.uuid4()}"
|
||||
ctx = (
|
||||
_pg_saver(thread_id)
|
||||
if backend == "postgres"
|
||||
else contextlib.nullcontext(MemorySaver())
|
||||
)
|
||||
with ctx as saver:
|
||||
graph = _scripted_graph(freq, profile, saver)
|
||||
config = {"configurable": {"thread_id": thread_id}}
|
||||
warm_config = {"configurable": {"thread_id": f"{thread_id}-warmup"}}
|
||||
try:
|
||||
warm_graph = _scripted_graph(freq, profile, saver)
|
||||
warm_graph.invoke({}, warm_config)
|
||||
warm_graph.get_state(warm_config)
|
||||
finally:
|
||||
saver.delete_thread(warm_config["configurable"]["thread_id"])
|
||||
reads: list[float] = []
|
||||
writes: list[float] = []
|
||||
slowest_update = 0
|
||||
slowest_config = config
|
||||
peak_read = -1.0
|
||||
try:
|
||||
for update in range(1, max(lengths) + 1):
|
||||
start = time.perf_counter()
|
||||
graph.invoke({}, config)
|
||||
writes.append((time.perf_counter() - start) * 1000)
|
||||
start = time.perf_counter()
|
||||
snapshot = graph.get_state(config)
|
||||
elapsed = (time.perf_counter() - start) * 1000
|
||||
reads.append(elapsed)
|
||||
if elapsed > peak_read:
|
||||
peak_read = elapsed
|
||||
slowest_update = update
|
||||
slowest_config = snapshot.config
|
||||
if update not in lengths:
|
||||
continue
|
||||
storage = (
|
||||
_postgres_storage_bytes(saver, thread_id)
|
||||
if backend == "postgres"
|
||||
else _memory_storage_bytes(saver, thread_id)
|
||||
)
|
||||
gc.collect()
|
||||
tracemalloc.start()
|
||||
try:
|
||||
graph.get_state(slowest_config)
|
||||
_, peak = tracemalloc.get_traced_memory()
|
||||
finally:
|
||||
tracemalloc.stop()
|
||||
yield {
|
||||
"backend": backend,
|
||||
"profile": profile,
|
||||
"frequency": freq,
|
||||
"updates": update,
|
||||
"read_samples": len(reads),
|
||||
"read_p50_ms": statistics.median(reads),
|
||||
"read_p95_ms": _percentile(reads, 0.95),
|
||||
"read_max_ms": max(reads),
|
||||
"slowest_update": slowest_update,
|
||||
"slowest_checkpoint_id": slowest_config["configurable"][
|
||||
"checkpoint_id"
|
||||
],
|
||||
"write_p50_ms": statistics.median(writes),
|
||||
"write_p95_ms": _percentile(writes, 0.95),
|
||||
"write_max_ms": max(writes),
|
||||
"write_total_s": sum(writes) / 1000,
|
||||
"storage_bytes": storage,
|
||||
"slowest_read_peak_heap_bytes": peak,
|
||||
}
|
||||
finally:
|
||||
saver.delete_thread(thread_id)
|
||||
|
||||
|
||||
def _pareto_knee(rows: list[dict], latency: str) -> dict:
|
||||
"""Pick the nondominated point nearest the min-max normalized ideal."""
|
||||
frontier = [
|
||||
r
|
||||
for r in rows
|
||||
if not any(
|
||||
float(o["storage_bytes"]) <= float(r["storage_bytes"])
|
||||
and float(o[latency]) <= float(r[latency])
|
||||
and (
|
||||
float(o["storage_bytes"]) < float(r["storage_bytes"])
|
||||
or float(o[latency]) < float(r[latency])
|
||||
)
|
||||
for o in rows
|
||||
)
|
||||
]
|
||||
bounds = {
|
||||
k: (min(float(r[k]) for r in rows), max(float(r[k]) for r in rows))
|
||||
for k in ("storage_bytes", latency)
|
||||
}
|
||||
|
||||
def score(row: dict) -> float:
|
||||
return sum(
|
||||
((float(row[k]) - lo) / (hi - lo or 1)) ** 2
|
||||
for k, (lo, hi) in bounds.items()
|
||||
)
|
||||
|
||||
return min(frontier, key=score)
|
||||
|
||||
|
||||
def _plot_sweep(csv_path: Path) -> None:
|
||||
"""Render dependency-free SVG frequency and Pareto plots for each group."""
|
||||
with csv_path.open() as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
groups = sorted({(r["backend"], r["profile"], r["updates"]) for r in rows})
|
||||
for backend, profile, updates in groups:
|
||||
group = sorted(
|
||||
(
|
||||
r
|
||||
for r in rows
|
||||
if (r["backend"], r["profile"], r["updates"])
|
||||
== (backend, profile, updates)
|
||||
),
|
||||
key=lambda r: int(r["frequency"]),
|
||||
)
|
||||
svg = [
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="420" viewBox="0 0 1100 420"><rect width="1100" height="420" fill="white"/>'
|
||||
]
|
||||
svg.append(
|
||||
f'<text x="20" y="25" font-family="sans-serif" font-size="16">{backend} / {profile} / {updates} updates (n={len(group)} frequencies)</text>'
|
||||
)
|
||||
for panel, (xkey, ykeys, title) in enumerate(
|
||||
[
|
||||
(
|
||||
"frequency",
|
||||
["storage_bytes", "read_p50_ms", "read_max_ms"],
|
||||
"Frequency vs independently normalized metrics",
|
||||
),
|
||||
(
|
||||
"storage_bytes",
|
||||
["read_p50_ms", "read_max_ms"],
|
||||
"Storage vs latency (Pareto tradeoff)",
|
||||
),
|
||||
]
|
||||
):
|
||||
left = 60 + panel * 550
|
||||
svg.append(
|
||||
f'<path d="M {left} 60 V 340 H {left + 450}" fill="none" stroke="black"/>'
|
||||
)
|
||||
svg.append(
|
||||
f'<text x="{left}" y="50" font-family="sans-serif" font-size="12">{title}</text>'
|
||||
)
|
||||
xs = [float(r[xkey]) for r in group]
|
||||
xmin, xmax = min(xs), max(xs)
|
||||
ymax = max(float(r[k]) for r in group for k in ykeys)
|
||||
for color, key in zip(["#176baf", "#c14b14", "#6a378b"], ykeys):
|
||||
scale = max(float(r[key]) for r in group) if panel == 0 else ymax
|
||||
|
||||
def coords(row: dict, scale: float = scale) -> tuple[float, float]:
|
||||
return left + (float(row[xkey]) - xmin) / (
|
||||
xmax - xmin or 1
|
||||
) * 450, 340 - float(row[key]) / (scale or 1) * 260
|
||||
|
||||
points = " ".join(f"{x:.2f},{y:.2f}" for x, y in map(coords, group))
|
||||
svg.append(
|
||||
f'<polyline points="{points}" fill="none" stroke="{color}" stroke-width="2"/>'
|
||||
)
|
||||
if panel == 1:
|
||||
knee = _pareto_knee(group, key)
|
||||
x, y = coords(knee)
|
||||
svg.append(
|
||||
f'<circle cx="{x}" cy="{y}" r="5" fill="{color}"/><text x="{x + 6}" y="{y - 8}" font-family="sans-serif" font-size="11">f={knee["frequency"]}</text>'
|
||||
)
|
||||
svg.append(
|
||||
f'<text x="{left}" y="{365 + ykeys.index(key) * 15}" fill="{color}" font-family="sans-serif" font-size="11">{key}</text>'
|
||||
)
|
||||
svg.append(
|
||||
f'<text x="{left}" y="355" font-family="sans-serif" font-size="10">{xkey}: {xmin:.0f} to {xmax:.0f}; y: 0 to {1 if panel == 0 else ymax:.2f}{" normalized" if panel == 0 else " ms"}</text>'
|
||||
)
|
||||
svg.append("</svg>")
|
||||
csv_path.with_name(
|
||||
f"{csv_path.stem}-{backend}-{profile}-{updates}.svg"
|
||||
).write_text("".join(svg))
|
||||
|
||||
|
||||
def run_snapshot_sweep(args: argparse.Namespace) -> None:
|
||||
"""Append completed grid points and regenerate plots after every point."""
|
||||
if args.backend == "postgres" and not _POSTGRES_AVAILABLE:
|
||||
raise RuntimeError("Postgres dependencies unavailable; no fallback is allowed")
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
metadata_path = args.output.with_suffix(".metadata.json")
|
||||
metadata = {
|
||||
"schema": 1,
|
||||
"python": platform.python_version(),
|
||||
"platform": platform.platform(),
|
||||
"script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
||||
"revision": subprocess.check_output(
|
||||
["git", "-C", str(Path(__file__).resolve().parent), "rev-parse", "HEAD"],
|
||||
text=True,
|
||||
).strip(),
|
||||
"backend": args.backend,
|
||||
"profiles": args.profiles,
|
||||
"frequencies": args.frequencies,
|
||||
"lengths": args.lengths,
|
||||
"script": "deterministic, no provider; IDs follow update index",
|
||||
"read_sampling": "one get_state after every invocation, prefix distribution",
|
||||
"write_sampling": "end-to-end graph.invoke including hydrate, scripted model and checkpoint I/O",
|
||||
"storage": "postgres pg_column_size row estimate (not relation disk bytes); memory serialized payloads across all stores",
|
||||
"max_supersteps_since_snapshot": os.environ.get(
|
||||
"LANGGRAPH_DELTA_MAX_SUPERSTEPS_SINCE_SNAPSHOT", "5000"
|
||||
),
|
||||
}
|
||||
if metadata_path.exists() and json.loads(metadata_path.read_text()) != metadata:
|
||||
raise ValueError("Resume metadata differs; use a new output path")
|
||||
metadata_path.write_text(json.dumps(metadata, indent=2) + "\n")
|
||||
existing = []
|
||||
if args.output.exists():
|
||||
with args.output.open() as handle:
|
||||
existing = list(csv.DictReader(handle))
|
||||
done = {(r["profile"], int(r["frequency"]), int(r["updates"])) for r in existing}
|
||||
for profile in args.profiles:
|
||||
for freq in args.frequencies:
|
||||
missing = [n for n in args.lengths if (profile, freq, n) not in done]
|
||||
if not missing:
|
||||
continue
|
||||
for row in _measure_sweep(freq, missing, profile, args.backend):
|
||||
fresh = not args.output.exists() or args.output.stat().st_size == 0
|
||||
with args.output.open("a", newline="") as handle:
|
||||
writer = csv.DictWriter(
|
||||
handle, fieldnames=list(row), lineterminator="\n"
|
||||
)
|
||||
if fresh:
|
||||
writer.writeheader()
|
||||
writer.writerow(row)
|
||||
handle.flush()
|
||||
print(json.dumps(row), flush=True)
|
||||
_plot_sweep(args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("DeltaChannel benchmark — multi-channel reads, mixed frequencies")
|
||||
print("=" * 78)
|
||||
run_benchmark()
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--snapshot-sweep", action="store_true")
|
||||
parser.add_argument("--backend", choices=["memory", "postgres"], default="memory")
|
||||
parser.add_argument("--frequencies", type=int, nargs="+", default=FREQUENCIES)
|
||||
parser.add_argument("--lengths", type=int, nargs="+", default=LENGTHS)
|
||||
parser.add_argument("--profiles", choices=PROFILES, nargs="+", default=PROFILES)
|
||||
parser.add_argument("--output", type=Path, default=Path("delta-sweep.csv"))
|
||||
args = parser.parse_args()
|
||||
if any(value <= 0 for value in args.frequencies + args.lengths):
|
||||
parser.error("frequencies and lengths must be positive")
|
||||
if args.snapshot_sweep:
|
||||
run_snapshot_sweep(args)
|
||||
else:
|
||||
run_benchmark()
|
||||
sys.exit(0)
|
||||
@@ -0,0 +1,104 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"delta_distributed",
|
||||
Path(__file__).resolve().parents[1] / "bench/delta_distributed.py",
|
||||
)
|
||||
bench = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(bench)
|
||||
|
||||
|
||||
def test_plan_grid():
|
||||
args = SimpleNamespace(
|
||||
snapshot_id="snapshot",
|
||||
concurrency=4,
|
||||
timeout=3600,
|
||||
vcpus=2,
|
||||
memory_gib=8,
|
||||
disk_gib=30,
|
||||
lengths=list(bench.LENGTHS),
|
||||
profiles=list(bench.PROFILES),
|
||||
frequencies=list(bench.FREQUENCIES),
|
||||
)
|
||||
plan = bench.make_plan(args)
|
||||
assert len(plan["shards"]) == 40
|
||||
assert len({s["key"] for s in plan["shards"]}) == 40
|
||||
assert plan["mem_bytes"] == 8 * 1024**3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("command_fails", [False, True])
|
||||
def test_shard_cleanup_resume_and_validation(tmp_path, command_fails):
|
||||
shard = {"key": "mock700-25", "profile": "mock700", "frequency": 25}
|
||||
plan = {
|
||||
"snapshot_id": "test",
|
||||
"timeout_seconds": 10,
|
||||
"vcpus": 2,
|
||||
"mem_bytes": 1024,
|
||||
"fs_capacity_bytes": 1024,
|
||||
"lengths": [50],
|
||||
"source_sha256": "hash",
|
||||
}
|
||||
data = b"backend,profile,frequency,updates,read_p50_ms,read_p95_ms,read_max_ms,storage_bytes\npostgres,mock700,25,50,1,2,3,400\n"
|
||||
deleted = []
|
||||
|
||||
class FakeSandbox:
|
||||
def write(self, path, content):
|
||||
assert "API_KEY" not in str(content)
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
return SimpleNamespace(exit_code=1 if command_fails else 0)
|
||||
|
||||
def read(self, path):
|
||||
return (
|
||||
json.dumps({"script_sha256": "hash"}).encode()
|
||||
if path.endswith("json")
|
||||
else data
|
||||
)
|
||||
|
||||
class FakeClient:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
pass
|
||||
|
||||
def create_sandbox(self, **kwargs):
|
||||
return FakeSandbox()
|
||||
|
||||
def delete_sandbox(self, name):
|
||||
deleted.append(name)
|
||||
|
||||
if command_fails:
|
||||
with pytest.raises(RuntimeError, match="Shard command failed"):
|
||||
bench.run_shard(plan, shard, tmp_path, FakeClient)
|
||||
assert len(deleted) == 1
|
||||
assert not (tmp_path / shard["key"] / "success.json").exists()
|
||||
return
|
||||
bench.run_shard(plan, shard, tmp_path, FakeClient)
|
||||
bench.run_shard(plan, shard, tmp_path, FakeClient)
|
||||
assert len(deleted) == 1
|
||||
assert (
|
||||
json.loads((tmp_path / shard["key"] / "attempt.json").read_text())["state"]
|
||||
== "deleted"
|
||||
)
|
||||
with pytest.raises(ValueError, match="Incomplete"):
|
||||
bench.validate_csv(data, shard, [50, 200])
|
||||
with pytest.raises(ValueError, match="Foreign"):
|
||||
bench.validate_csv(data, {**shard, "frequency": 50}, [50])
|
||||
|
||||
|
||||
def test_failed_shard_is_deleted(tmp_path):
|
||||
shard = {"key": "mock700-25", "profile": "mock700", "frequency": 25}
|
||||
target = tmp_path / shard["key"]
|
||||
target.mkdir()
|
||||
(target / "attempt.json").write_text(
|
||||
json.dumps({"state": "running", "sandbox_name": "previous"})
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="previous sandbox"):
|
||||
bench.run_shard({}, shard, tmp_path)
|
||||
assert not (target / "success.json").exists()
|
||||
@@ -0,0 +1,48 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import create_autospec
|
||||
|
||||
import pytest
|
||||
from langsmith.sandbox import Sandbox, SandboxClient
|
||||
|
||||
SPEC = importlib.util.spec_from_file_location(
|
||||
"delta_snapshot", Path(__file__).resolve().parents[1] / "bench/delta_snapshot.py"
|
||||
)
|
||||
builder = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(builder)
|
||||
|
||||
|
||||
def test_snapshot_build_contract():
|
||||
args = SimpleNamespace(
|
||||
name="delta-bench",
|
||||
revision="a" * 40,
|
||||
vcpus=2,
|
||||
memory_gib=8,
|
||||
disk_gib=30,
|
||||
timeout=1800,
|
||||
)
|
||||
plan = builder.snapshot_plan(args)
|
||||
client = create_autospec(SandboxClient, instance=True)
|
||||
client.__enter__.return_value = client
|
||||
client.create_snapshot_from_dockerfile.return_value = SimpleNamespace(
|
||||
id="snapshot-id", status="ready"
|
||||
)
|
||||
receipt = builder.build_snapshot(plan, lambda: client)
|
||||
assert receipt["snapshot_id"] == "snapshot-id"
|
||||
call = client.create_snapshot_from_dockerfile.call_args.kwargs
|
||||
assert call["build_args"] == {"LANGGRAPH_REVISION": "a" * 40}
|
||||
assert sorted(p.name for p in call["context"].iterdir()) == ["Dockerfile"]
|
||||
assert call["timeout"] == 1800
|
||||
client.__exit__.assert_called_once()
|
||||
args.revision = "main; echo nope"
|
||||
with pytest.raises(ValueError, match="40-character"):
|
||||
builder.snapshot_plan(args)
|
||||
|
||||
|
||||
def test_sandbox_sdk_file_api():
|
||||
sandbox = create_autospec(Sandbox, instance=True)
|
||||
sandbox.write("/tmp/test", b"payload")
|
||||
sandbox.read("/tmp/test")
|
||||
with pytest.raises(AttributeError):
|
||||
sandbox.write_file("/tmp/test", b"payload")
|
||||
@@ -202,11 +202,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.1.4"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1919,7 +1919,7 @@ requires-dist = [
|
||||
{ name = "langchain-core", specifier = ">=1.4.0,<2" },
|
||||
{ name = "langchain-protocol", specifier = ">=0.0.15" },
|
||||
{ name = "orjson", specifier = ">=3.11.5" },
|
||||
{ name = "websockets", specifier = ">=14,<17" },
|
||||
{ name = "websockets", specifier = ">=14" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
|
||||
@@ -49,11 +49,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.1.4"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e0/2d/a891ca51311197f6ad14a7ef42e2399f36cf2f9bd44752b3dc4eab60fdc5/certifi-2026.1.4.tar.gz", hash = "sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120", size = 154268, upload-time = "2026-01-04T02:42:41.825Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e6/ad/3cc14f097111b4de0040c83a525973216457bbeeb63739ef1ed275c1c021/certifi-2026.1.4-py3-none-any.whl", hash = "sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c", size = 152900, upload-time = "2026-01-04T02:42:40.15Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -614,7 +614,7 @@ requires-dist = [
|
||||
{ name = "langchain-core", specifier = ">=1.4.0,<2" },
|
||||
{ name = "langchain-protocol", specifier = ">=0.0.15" },
|
||||
{ name = "orjson", specifier = ">=3.11.5" },
|
||||
{ name = "websockets", specifier = ">=14,<17" },
|
||||
{ name = "websockets", specifier = ">=14" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
|
||||
@@ -34,6 +34,14 @@
|
||||
- **`thread.output`** — awaitable that resolves to the terminal thread state
|
||||
`values` dict after the run lifecycle completes.
|
||||
|
||||
- **`Priority` (beta)** — register an async function with `@priority.assign`
|
||||
and name the object in the `priority` section of `langgraph.json`. The
|
||||
server calls the function with a `PriorityContext` at the first run on a
|
||||
thread with no settings and stores the `PrioritySettings` it returns
|
||||
(`priority`, `fairness_group`, `fairness_weight`) on the thread. Requires
|
||||
`langgraph-api` 0.18.0 or later and a `langgraph-cli` that supports the
|
||||
section; older versions of either ignore it.
|
||||
|
||||
### Changed
|
||||
|
||||
- `client.threads.stream()` now accepts `transport="sse"` (default) or
|
||||
|
||||
@@ -2,14 +2,19 @@ from langgraph_sdk.auth import Auth
|
||||
from langgraph_sdk.client import get_client, get_sync_client
|
||||
from langgraph_sdk.encryption import Encryption
|
||||
from langgraph_sdk.encryption.types import DecryptResult, EncryptionContext
|
||||
from langgraph_sdk.priority import Priority
|
||||
from langgraph_sdk.priority.types import PriorityContext, PrioritySettings
|
||||
|
||||
__version__ = "0.4.8"
|
||||
__version__ = "0.4.9"
|
||||
|
||||
__all__ = [
|
||||
"Auth",
|
||||
"DecryptResult",
|
||||
"Encryption",
|
||||
"EncryptionContext",
|
||||
"Priority",
|
||||
"PriorityContext",
|
||||
"PrioritySettings",
|
||||
"get_client",
|
||||
"get_sync_client",
|
||||
]
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Custom run priority for LangGraph.
|
||||
|
||||
.. warning::
|
||||
This API is in beta and may change in future versions.
|
||||
|
||||
This module lets developers decide the priority settings of each thread with
|
||||
an async function that the server calls at the first run on a thread with no
|
||||
settings. The function is registered on an object that `langgraph.json` names,
|
||||
and runs server-side.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import inspect
|
||||
import typing
|
||||
import warnings
|
||||
from collections.abc import Awaitable, Callable
|
||||
|
||||
from langgraph_sdk.encryption import DuplicateHandlerError, LangGraphBetaWarning
|
||||
from langgraph_sdk.priority import types
|
||||
|
||||
_AssignHandler = Callable[
|
||||
[types.PriorityContext], Awaitable[types.PrioritySettings | None]
|
||||
]
|
||||
_AssignHandlerT = typing.TypeVar("_AssignHandlerT", bound=_AssignHandler)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _warn_priority_beta() -> None:
|
||||
warnings.warn(
|
||||
"The Priority API is in beta and may change in future versions.",
|
||||
LangGraphBetaWarning,
|
||||
stacklevel=4,
|
||||
)
|
||||
|
||||
|
||||
def _validate_handler(fn: typing.Callable) -> None:
|
||||
"""Validate that the assign handler is async and takes one parameter.
|
||||
|
||||
Raises:
|
||||
TypeError: If the handler is not an async function or has wrong parameter count
|
||||
"""
|
||||
if not inspect.iscoroutinefunction(fn):
|
||||
raise TypeError(
|
||||
f"Priority assign handler must be an async function, got {type(fn)}"
|
||||
)
|
||||
|
||||
sig = inspect.signature(fn)
|
||||
params = [
|
||||
p
|
||||
for p in sig.parameters.values()
|
||||
if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)
|
||||
]
|
||||
if len(params) != 1:
|
||||
raise TypeError(
|
||||
"Priority assign handler must accept exactly 1 parameter (ctx), "
|
||||
f"got {len(params)}"
|
||||
)
|
||||
|
||||
|
||||
class Priority:
|
||||
"""Decide the priority settings of your LangGraph application's threads.
|
||||
|
||||
.. warning::
|
||||
This API is in beta and may change in future versions.
|
||||
|
||||
To use, create a separate Python file and add the path to the file to your
|
||||
LangGraph API configuration file (`langgraph.json`). Within that file, create
|
||||
an instance of the Priority class and register an assign function with
|
||||
`@priority.assign`.
|
||||
|
||||
Example `langgraph.json` file:
|
||||
|
||||
```json
|
||||
{
|
||||
"dependencies": ["."],
|
||||
"graphs": {
|
||||
"agent": "./my_agent/agent.py:graph"
|
||||
},
|
||||
"env": ".env",
|
||||
"priority": {
|
||||
"path": "./priority.py:priority"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The server calls the assign function at the first run on a thread with no
|
||||
settings, which can come long after the thread was created, and stores the
|
||||
settings it returns on the thread. Later runs on the thread use the stored
|
||||
settings. The function can be called more than once for one thread, for
|
||||
example after it raises, and only one result is stored, so keep it free of
|
||||
side effects.
|
||||
|
||||
???+ example "Basic Usage"
|
||||
|
||||
```python
|
||||
from langgraph_sdk import Priority, PriorityContext, PrioritySettings
|
||||
|
||||
priority = Priority()
|
||||
|
||||
@priority.assign
|
||||
async def assign(ctx: PriorityContext) -> PrioritySettings | None:
|
||||
return {
|
||||
"priority": 3 if "premium" in ctx.permissions else 0,
|
||||
# Stored unencrypted, so an opaque ID rather than personal data.
|
||||
"fairness_group": ctx.user["org_id"],
|
||||
}
|
||||
```
|
||||
"""
|
||||
|
||||
__slots__ = ("_assign_handler",)
|
||||
|
||||
types = types
|
||||
"""Reference to priority type definitions.
|
||||
|
||||
Provides access to the type definitions used by the priority system:
|
||||
PriorityContext and PrioritySettings.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize the Priority instance."""
|
||||
_warn_priority_beta()
|
||||
self._assign_handler: _AssignHandler | None = None
|
||||
|
||||
def assign(self, fn: _AssignHandlerT) -> _AssignHandlerT:
|
||||
"""Register the function that assigns a thread's priority settings.
|
||||
|
||||
The function receives a `PriorityContext` and returns `PrioritySettings`,
|
||||
or `None` for the defaults.
|
||||
|
||||
Example:
|
||||
```python
|
||||
@priority.assign
|
||||
async def assign(ctx: PriorityContext) -> PrioritySettings | None:
|
||||
if ctx.cron_id is not None:
|
||||
return {"priority": -2}
|
||||
return None
|
||||
```
|
||||
|
||||
Args:
|
||||
fn: The assign handler function
|
||||
|
||||
Returns:
|
||||
The registered handler function
|
||||
|
||||
Raises:
|
||||
DuplicateHandlerError: If an assign handler is already registered
|
||||
TypeError: If the handler has an invalid signature
|
||||
"""
|
||||
if self._assign_handler is not None:
|
||||
raise DuplicateHandlerError("Priority assign handler already registered")
|
||||
_validate_handler(fn)
|
||||
self._assign_handler = fn
|
||||
return fn
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Priority types for LangGraph.
|
||||
|
||||
This module defines the context the server passes to the function registered
|
||||
with `@priority.assign`, and the settings that function returns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from langgraph_sdk.auth.types import BaseUser
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PriorityContext:
|
||||
"""Context passed to the `@priority.assign` function at the first run on a thread with no settings."""
|
||||
|
||||
user: BaseUser
|
||||
"""The user from the deployment's auth, custom auth or LangSmith auth. Without auth, `user.identity` is empty."""
|
||||
|
||||
permissions: Sequence[str]
|
||||
"""The user's permissions from the same auth. Without auth, it's empty."""
|
||||
|
||||
thread_id: str | None
|
||||
"""The thread's ID, or `None` when the run creates its thread without being given an ID."""
|
||||
|
||||
thread_metadata: dict[str, typing.Any]
|
||||
"""The thread's metadata."""
|
||||
|
||||
assistant_id: str
|
||||
"""The ID of the assistant the run uses."""
|
||||
|
||||
graph_id: str
|
||||
"""The ID of the graph the assistant runs."""
|
||||
|
||||
assistant_metadata: dict[str, typing.Any]
|
||||
"""The assistant's metadata."""
|
||||
|
||||
run_metadata: dict[str, typing.Any]
|
||||
"""The run's metadata."""
|
||||
|
||||
cron_id: str | None
|
||||
"""The ID of the cron that started the run, or `None` when no cron started it."""
|
||||
|
||||
|
||||
class PrioritySettings(typing.TypedDict, total=False):
|
||||
"""Priority settings returned by the `@priority.assign` function.
|
||||
|
||||
Every key is optional: a missing key or a key set to `None` takes its
|
||||
default, and returning `None` instead of a dict gives every key its default.
|
||||
A value outside its range, or an empty `fairness_group`, is an error.
|
||||
"""
|
||||
|
||||
priority: int | None
|
||||
"""An integer from -5 to 5, where 5 runs first and -5 runs last. Defaults to 0."""
|
||||
|
||||
fairness_group: str | None
|
||||
"""The thread's fairness group, a non-empty string. Defaults to `"default"`.
|
||||
|
||||
The server stores it unencrypted, even when the deployment uses custom
|
||||
encryption, so use an opaque ID rather than personal data.
|
||||
"""
|
||||
|
||||
fairness_weight: int | None
|
||||
"""The weight of the thread's fairness group, an integer from 1 to 100. Defaults to 1."""
|
||||
@@ -16,7 +16,7 @@ dependencies = [
|
||||
"orjson>=3.11.5",
|
||||
"langchain-protocol>=0.0.15",
|
||||
"langchain-core>=1.4.0,<2",
|
||||
"websockets>=14,<17",
|
||||
"websockets>=14",
|
||||
]
|
||||
|
||||
[tool.hatch.version]
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
import langgraph_sdk
|
||||
from langgraph_sdk import Priority, PriorityContext, PrioritySettings
|
||||
from langgraph_sdk.encryption import DuplicateHandlerError, LangGraphBetaWarning
|
||||
from langgraph_sdk.priority import _warn_priority_beta
|
||||
|
||||
|
||||
class _User:
|
||||
is_authenticated = True
|
||||
display_name = "user-1"
|
||||
identity = "user-1"
|
||||
permissions: tuple[str, ...] = ()
|
||||
|
||||
def __getitem__(self, key):
|
||||
return getattr(self, key)
|
||||
|
||||
def __contains__(self, key):
|
||||
return hasattr(self, key)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(("identity", "display_name", "is_authenticated", "permissions"))
|
||||
|
||||
|
||||
def _context(thread_id: str | None) -> PriorityContext:
|
||||
return PriorityContext(
|
||||
user=_User(),
|
||||
permissions=[],
|
||||
thread_id=thread_id,
|
||||
thread_metadata={},
|
||||
assistant_id="assistant-1",
|
||||
graph_id="agent",
|
||||
assistant_metadata={"tier": "gold"},
|
||||
run_metadata={},
|
||||
cron_id=None,
|
||||
)
|
||||
|
||||
|
||||
async def _no_params() -> PrioritySettings | None:
|
||||
return None
|
||||
|
||||
|
||||
async def _two_params(_ctx: PriorityContext, _extra: object) -> PrioritySettings | None:
|
||||
return None
|
||||
|
||||
|
||||
def test_priority_and_its_types_are_exported():
|
||||
assert {"Priority", "PriorityContext", "PrioritySettings"} <= set(
|
||||
langgraph_sdk.__all__
|
||||
)
|
||||
assert Priority.types.PriorityContext is PriorityContext
|
||||
assert Priority.types.PrioritySettings is PrioritySettings
|
||||
|
||||
|
||||
def test_construction_warns_that_priority_is_beta():
|
||||
# The warning fires once per process, so an earlier test may have used it up.
|
||||
_warn_priority_beta.cache_clear()
|
||||
with pytest.warns(LangGraphBetaWarning, match="Priority API is in beta"):
|
||||
Priority()
|
||||
|
||||
|
||||
def test_assign_registers_and_returns_the_function():
|
||||
priority = Priority()
|
||||
|
||||
async def assign(_ctx: PriorityContext) -> PrioritySettings | None:
|
||||
return {"priority": 3, "fairness_group": "org-42"}
|
||||
|
||||
assert priority.assign(assign) is assign
|
||||
assert priority._assign_handler is assign
|
||||
|
||||
|
||||
def test_assign_rejects_a_sync_function():
|
||||
priority = Priority()
|
||||
|
||||
def assign(_ctx: PriorityContext) -> PrioritySettings | None:
|
||||
return None
|
||||
|
||||
with pytest.raises(TypeError, match="must be an async function"):
|
||||
priority.assign(assign) # ty: ignore[invalid-argument-type]
|
||||
assert priority._assign_handler is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fn", [_no_params, _two_params])
|
||||
def test_assign_rejects_the_wrong_number_of_parameters(fn):
|
||||
priority = Priority()
|
||||
|
||||
with pytest.raises(TypeError, match="must accept exactly 1 parameter"):
|
||||
priority.assign(fn)
|
||||
assert priority._assign_handler is None
|
||||
|
||||
|
||||
def test_assign_rejects_a_second_function():
|
||||
priority = Priority()
|
||||
|
||||
@priority.assign
|
||||
async def first(_ctx: PriorityContext) -> PrioritySettings | None:
|
||||
return None
|
||||
|
||||
with pytest.raises(DuplicateHandlerError):
|
||||
|
||||
@priority.assign
|
||||
async def second(_ctx: PriorityContext) -> PrioritySettings | None:
|
||||
return None
|
||||
|
||||
assert priority._assign_handler is first
|
||||
|
||||
|
||||
def test_context_accepts_no_thread_id():
|
||||
# `make lint` type-checks this call too, so `thread_id` must accept None there.
|
||||
assert _context(thread_id=None).thread_id is None
|
||||
|
||||
|
||||
def test_context_is_frozen():
|
||||
ctx = _context(thread_id="thread-1")
|
||||
|
||||
with pytest.raises(dataclasses.FrozenInstanceError):
|
||||
ctx.thread_id = "thread-2" # ty: ignore[invalid-assignment]
|
||||
|
||||
|
||||
def test_every_settings_key_is_optional():
|
||||
assert PrioritySettings.__optional_keys__ == {
|
||||
"priority",
|
||||
"fairness_group",
|
||||
"fairness_weight",
|
||||
}
|
||||
assert PrioritySettings.__required_keys__ == frozenset()
|
||||
@@ -1,5 +1,5 @@
|
||||
version = 1
|
||||
revision = 5
|
||||
revision = 3
|
||||
requires-python = ">=3.10"
|
||||
resolution-markers = [
|
||||
"python_full_version >= '3.15'",
|
||||
@@ -40,11 +40,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "certifi"
|
||||
version = "2026.2.25"
|
||||
version = "2026.10.19"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -525,7 +525,7 @@ requires-dist = [
|
||||
{ name = "langchain-core", specifier = ">=1.4.0,<2" },
|
||||
{ name = "langchain-protocol", specifier = ">=0.0.15" },
|
||||
{ name = "orjson", specifier = ">=3.11.5" },
|
||||
{ name = "websockets", specifier = ">=14,<17" },
|
||||
{ name = "websockets", specifier = ">=14" },
|
||||
]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
|
||||