Compare commits

...
3 changed files with 273 additions and 35 deletions
+234 -29
View File
@@ -16,6 +16,7 @@ DEFAULT_PYTHON_VERSION = "3.11"
DEFAULT_IMAGE_DISTRO = "debian"
CONSTRAINTS_PATH = "/api/constraints.txt"
Distros = Literal["debian", "wolfi", "bullseye", "bookworm"]
MiddlewareOrders = Literal["auth_first", "middleware_first"]
@@ -793,6 +794,15 @@ def validate_config_file(config_path: pathlib.Path) -> Config:
return validated
class ReqGenSpec(NamedTuple):
host_pkg_path: pathlib.Path
container_pkg_path: str
container_req_path: str
package_type: Literal["pyproject", "setup"]
has_uv_lock: bool
stage_name: Optional[str]
class LocalDeps(NamedTuple):
"""A container for referencing and managing local Python dependencies.
@@ -840,6 +850,15 @@ class LocalDeps(NamedTuple):
additional_contexts: A list of paths to directories that contain local
dependencies in parent directories. These directories are added to the
Docker build context to ensure that the Dockerfile can access them.
pkgs_missing_reqs: A list of packages that need requirements.txt generated.
Each entry is a ReqGenSpec containing:
- host_pkg_path: Absolute host path of the package directory
- container_pkg_path: Target path in the container for metadata/requirements
- container_req_path: Full path to requirements.txt inside the container
- package_type: "pyproject" or "setup"
- has_uv_lock: True if the package includes uv.lock
- stage_name: BuildKit context name if outside the build context, else None
"""
pip_reqs: list[tuple[pathlib.Path, str]]
@@ -848,7 +867,9 @@ class LocalDeps(NamedTuple):
# if . is in dependencies, use it as working_dir
working_dir: Optional[str] = None
# if there are local dependencies in parent directories, use additional_contexts
additional_contexts: list[pathlib.Path] = None
additional_contexts: Optional[list[pathlib.Path]] = None
# Packages that need requirements.txt generation
pkgs_missing_reqs: Optional[list[ReqGenSpec]] = None
def _assemble_local_deps(config_path: pathlib.Path, config: Config) -> LocalDeps:
@@ -884,6 +905,7 @@ def _assemble_local_deps(config_path: pathlib.Path, config: Config) -> LocalDeps
faux_pkgs = {}
working_dir: Optional[str] = None
additional_contexts: list[pathlib.Path] = []
pkgs_missing_reqs: list[ReqGenSpec] = []
for local_dep in config["dependencies"]:
if not local_dep.startswith("."):
@@ -923,6 +945,35 @@ def _assemble_local_deps(config_path: pathlib.Path, config: Config) -> LocalDeps
# set working_dir
if local_dep == ".":
working_dir = f"/deps/{container_name}"
requirement_path = f"/deps/{container_name}/requirements.txt"
# Track packages needing requirements.txt generation (real packages only)
if "requirements.txt" not in files:
has_pyproject = "pyproject.toml" in files
has_uv_lock = "uv.lock" in files
if has_pyproject:
pkg_type = "pyproject"
has_lock = has_uv_lock
else: # setup.py only
pkg_type = "setup"
has_lock = False
container_pkg_path = f"/deps/{container_name}"
stage_name = None
if config_path.parent not in resolved.parents:
stage_name = container_name
pkgs_missing_reqs.append(
ReqGenSpec(
host_pkg_path=resolved,
container_pkg_path=container_pkg_path,
container_req_path=requirement_path,
package_type=pkg_type,
has_uv_lock=has_lock,
stage_name=stage_name,
)
)
else:
# We could not find a pyproject.toml or setup.py, so treat as a faux package
if any(file == "__init__.py" for file in files):
@@ -954,19 +1005,27 @@ def _assemble_local_deps(config_path: pathlib.Path, config: Config) -> LocalDeps
faux_pkgs[resolved] = (local_dep, container_path)
if local_dep == ".":
working_dir = container_path
requirement_path = f"{container_path}/requirements.txt"
# If the faux package has a requirements.txt, we'll add
# the path to the list of requirements to install.
if "requirements.txt" in files:
rfile = resolved / "requirements.txt"
pip_reqs.append(
(
rfile,
f"{container_path}/requirements.txt",
)
# If the package has a requirements.txt, we'll add
# the path to the list of requirements to install.
if "requirements.txt" in files:
rfile = resolved / "requirements.txt"
pip_reqs.append(
(
rfile,
requirement_path,
)
)
return LocalDeps(pip_reqs, real_pkgs, faux_pkgs, working_dir, additional_contexts)
return LocalDeps(
pip_reqs,
real_pkgs,
faux_pkgs,
working_dir,
additional_contexts,
pkgs_missing_reqs,
)
def _update_graph_paths(
@@ -1261,6 +1320,110 @@ def get_build_tools_to_uninstall(config: Config) -> tuple[str]:
)
def _metadata_files(spec: ReqGenSpec) -> list[str]:
files = ["pyproject.toml"] if spec.package_type == "pyproject" else ["setup.py"]
if spec.package_type == "pyproject" and spec.has_uv_lock:
files.append("uv.lock")
if spec.package_type == "setup" and (spec.host_pkg_path / "setup.cfg").exists():
files.append("setup.cfg")
return files
def _get_reqs_gen_cmd(spec: ReqGenSpec) -> str:
if spec.package_type == "pyproject" and spec.has_uv_lock:
return "uv export --no-hashes --no-dev --no-emit-local -o 'requirements.txt'"
if spec.package_type == "pyproject":
return f"uv pip compile pyproject.toml -o 'requirements.txt' --constraint {CONSTRAINTS_PATH}"
return (
f"uv pip compile setup.py -o 'requirements.txt' --constraint {CONSTRAINTS_PATH}"
)
def _generate_requirements_from_metadata(
config_path: pathlib.Path,
local_deps: LocalDeps,
pip_installer: Literal["uv", "pip"],
) -> str:
"""Generate requirements.txt from uv.lock, pyproject.toml, or setup.py.
This function creates Docker layers that:
1. Copy packaging metadata files (pyproject.toml, setup.py, uv.lock, etc.)
2. Generate requirements.txt using appropriate uv commands:
- `uv export` if uv.lock exists (exact locked versions)
- `uv pip compile` for pyproject.toml or setup.py (fresh resolution)
The generated requirements.txt files are then handled by the existing
pip_reqs installation logic, maintaining a single code path for all packages.
Supports:
- pyproject.toml with uv.lock → uv export (preserves lock file versions)
- pyproject.toml without uv.lock → uv pip compile (resolve dependencies)
- setup.py (with optional setup.cfg) → uv pip compile (resolve dependencies)
Args:
config_path: Path to the langgraph.json config file
local_deps: LocalDeps object containing package information
pip_installer: Either "uv" or "pip"
Returns:
Docker instruction string for requirements.txt generation,
or empty string if not applicable (pip installer or no packages to generate)
"""
if pip_installer != "uv" or not local_deps.pkgs_missing_reqs:
# if installer is pip, we need pip-tools to generate requirements.txt
# this doesn't come automatically with pip, so we need to install it
# in base images, but ci uses uv, which is where we need the caching
# so limit to uv.
return ""
docker_lines = ["# -- Generate requirements.txt for packages without one --"]
# Layer 1: Copy packaging metadata files needed for requirements generation
docker_lines.append("# Copy packaging metadata files")
for spec in sorted(
local_deps.pkgs_missing_reqs, key=lambda s: s.container_pkg_path
):
for file_name in _metadata_files(spec):
if (
local_deps.additional_contexts
and spec.host_pkg_path in local_deps.additional_contexts
):
if not spec.stage_name:
raise RuntimeError(
f"Package {spec.host_pkg_path} in additional_contexts but has no stage_name"
)
docker_lines.append(
f"COPY --from={spec.stage_name} {file_name} {spec.container_pkg_path}/{file_name}"
)
else:
file_relpath = (spec.host_pkg_path / file_name).relative_to(
config_path.parent
)
docker_lines.append(
f"ADD {file_relpath} {spec.container_pkg_path}/{file_name}"
)
# Layer 2: Generate requirements.txt files using appropriate uv commands
docker_lines.append("")
docker_lines.append("# Generate requirements.txt from packaging metadata")
for spec in sorted(
local_deps.pkgs_missing_reqs, key=lambda s: s.container_pkg_path
):
pkg_name = spec.host_pkg_path.name
if spec.package_type == "pyproject" and spec.has_uv_lock:
docker_lines.append(f"# Generate from uv.lock for {pkg_name}")
elif spec.package_type == "pyproject":
docker_lines.append(f"# Compile from pyproject.toml for {pkg_name}")
else:
docker_lines.append(f"# Compile from setup.py for {pkg_name}")
docker_lines.append(
f"RUN cd '{spec.container_pkg_path}' && {_get_reqs_gen_cmd(spec)}"
)
docker_lines.append("# -- End of requirements.txt generation --")
return os.linesep.join(docker_lines)
def python_config_to_docker(
config_path: pathlib.Path,
config: Config,
@@ -1283,8 +1446,12 @@ def python_config_to_docker(
raise ValueError(f"Invalid pip_installer: {pip_installer}")
# configure pip
local_reqs_pip_install = f"PYTHONDONTWRITEBYTECODE=1 {install_cmd} --no-cache-dir -c /api/constraints.txt"
global_reqs_pip_install = f"PYTHONDONTWRITEBYTECODE=1 {install_cmd} --no-cache-dir -c /api/constraints.txt"
local_reqs_pip_install = (
f"PYTHONDONTWRITEBYTECODE=1 {install_cmd} --no-cache-dir -c {CONSTRAINTS_PATH}"
)
global_reqs_pip_install = (
f"PYTHONDONTWRITEBYTECODE=1 {install_cmd} --no-cache-dir -c {CONSTRAINTS_PATH}"
)
if config.get("pip_config_file"):
local_reqs_pip_install = (
f"PIP_CONFIG_FILE=/pipconfig.txt {local_reqs_pip_install}"
@@ -1311,22 +1478,59 @@ def python_config_to_docker(
pip_pkgs_str = (
f"RUN {local_reqs_pip_install} {' '.join(pypi_deps)}" if pypi_deps else ""
)
if local_deps.pip_reqs:
pip_reqs_str = os.linesep.join(
(
f"COPY --from=outer-{reqpath.name} requirements.txt {destpath}"
if reqpath.parent in local_deps.additional_contexts
else f"ADD {reqpath.relative_to(config_path.parent)} {destpath}"
)
for reqpath, destpath in local_deps.pip_reqs
)
pip_reqs_str += f"{os.linesep}RUN {local_reqs_pip_install} {' '.join('-r ' + r for _, r in local_deps.pip_reqs)}"
pip_reqs_str = f"""# -- Installing local requirements --
{pip_reqs_str}
# -- End of local requirements install --"""
else:
pip_reqs_str = ""
# Generate requirements.txt layer for packages that need it
# This happens BEFORE copying existing requirements.txt files
generated_reqs_str = _generate_requirements_from_metadata(
config_path, local_deps, pip_installer
)
# Combine existing requirements.txt with generated ones in a single deterministic layer
all_req_paths: list[str] = []
copy_lines: list[str] = []
# Map additional_contexts path -> stage name
additional_ctx_stage: dict[pathlib.Path, str] = {}
for p in local_deps.additional_contexts or []:
if p in local_deps.real_pkgs:
additional_ctx_stage[p] = local_deps.real_pkgs[p][1]
elif p in local_deps.faux_pkgs:
additional_ctx_stage[p] = f"outer-{p.name}"
else:
raise RuntimeError(
f"Package {p} in additional_contexts but not in real_pkgs or faux_pkgs"
)
# Existing reqs
for reqpath, destpath in local_deps.pip_reqs or []:
if local_deps.additional_contexts and reqpath.parent in additional_ctx_stage:
copy_lines.append(
f"COPY --from={additional_ctx_stage[reqpath.parent]} requirements.txt {destpath}"
)
else:
copy_lines.append(
f"ADD {reqpath.relative_to(config_path.parent)} {destpath}"
)
all_req_paths.append(destpath)
# Generated reqs
if (
pip_installer == "uv"
): # we are only generate a requirements.txt if installer is uv (for now)
for spec in local_deps.pkgs_missing_reqs or []:
all_req_paths.append(spec.container_req_path)
pip_reqs_str = ""
if all_req_paths:
# Stabilize order
all_req_paths = sorted(set(all_req_paths))
# Install each requirements.txt sequentially. This mimics the previous sequential solver behavior
# which allows to adjust/downgrade packages between installs, so no conflict is raised.
pip_reqs_str = f"""# -- Installing from requirements.txt files --
{os.linesep.join(copy_lines)}
{os.linesep.join(f"RUN {local_reqs_pip_install} -r '{p}'" for p in all_req_paths)}
# -- End of requirements.txt install --"""
# generate lock file if real package and lock file missing
# https://setuptools.pypa.io/en/latest/userguide/datafiles.html#package-data
# https://til.simonwillison.net/python/pyproject
@@ -1380,6 +1584,7 @@ ADD {relpath} /deps/{name}
install_node_str,
pip_config_file_str,
pip_pkgs_str,
generated_reqs_str,
pip_reqs_str,
local_pkgs_str,
faux_pkgs_str,
@@ -1417,7 +1622,7 @@ ADD {relpath} /deps/{name}
[
"# -- Installing JS dependencies --",
f"ENV NODE_VERSION={config.get('node_version') or DEFAULT_NODE_VERSION}",
f"RUN cd {local_deps.working_dir} && {_get_node_pm_install_cmd(config_path, config)} && tsx /api/langgraph_api/js/build.mts",
f"RUN cd '{local_deps.working_dir}' && {_get_node_pm_install_cmd(config_path, config)} && tsx /api/langgraph_api/js/build.mts",
"# -- End of JS dependencies install --",
]
)
+15
View File
@@ -142,6 +142,21 @@ services:
- cli_1: {str(pathlib.Path(__file__).parent.parent.parent.parent.absolute())}
dockerfile_inline: |
FROM langchain/langgraph-api:3.11
# -- Generate requirements.txt for packages without one --
# Copy packaging metadata files
ADD pyproject.toml /deps/cli/pyproject.toml
COPY --from=cli_1 pyproject.toml /deps/cli_1/pyproject.toml
COPY --from=cli_1 uv.lock /deps/cli_1/uv.lock
# Generate requirements.txt from packaging metadata
# Compile from pyproject.toml for cli
RUN cd '/deps/cli' && uv pip compile pyproject.toml -o 'requirements.txt' --constraint /api/constraints.txt
# Generate from uv.lock for cli
RUN cd '/deps/cli_1' && uv export --no-hashes --no-dev -o 'requirements.txt'
# -- End of requirements.txt generation --
# -- Installing from requirements.txt files --
RUN PYTHONDONTWRITEBYTECODE=1 uv pip install --system --no-cache-dir -c /api/constraints.txt -r '/deps/cli/requirements.txt'
RUN PYTHONDONTWRITEBYTECODE=1 uv pip install --system --no-cache-dir -c /api/constraints.txt -r '/deps/cli_1/requirements.txt'
# -- End of requirements.txt install --
# -- Adding local package . --
ADD . /deps/cli
# -- End of local package . --
+24 -6
View File
@@ -420,10 +420,18 @@ def test_config_to_docker_simple():
)
expected_docker_stdin = f"""\
FROM langchain/langgraph-api:3.11
# -- Installing local requirements --
COPY --from=outer-requirements.txt requirements.txt /deps/outer-graphs_reqs_a/graphs_reqs_a/requirements.txt
RUN PYTHONDONTWRITEBYTECODE=1 uv pip install --system --no-cache-dir -c /api/constraints.txt -r /deps/outer-graphs_reqs_a/graphs_reqs_a/requirements.txt
# -- End of local requirements install --
# -- Generate requirements.txt for packages without one --
# Copy packaging metadata files
COPY --from=examples pyproject.toml /deps/examples/pyproject.toml
# Generate requirements.txt from packaging metadata
# Compile from pyproject.toml for examples
RUN cd '/deps/examples' && uv pip compile pyproject.toml -o 'requirements.txt' --constraint /api/constraints.txt
# -- End of requirements.txt generation --
# -- Installing from requirements.txt files --
COPY --from=outer-graphs_reqs_a requirements.txt /deps/outer-graphs_reqs_a/graphs_reqs_a/requirements.txt
RUN PYTHONDONTWRITEBYTECODE=1 uv pip install --system --no-cache-dir -c /api/constraints.txt -r '/deps/examples/requirements.txt'
RUN PYTHONDONTWRITEBYTECODE=1 uv pip install --system --no-cache-dir -c /api/constraints.txt -r '/deps/outer-graphs_reqs_a/graphs_reqs_a/requirements.txt'
# -- End of requirements.txt install --
# -- Adding local package ../../examples --
COPY --from=examples . /deps/examples
# -- End of local package ../../examples --
@@ -653,6 +661,16 @@ dependencies = ["langchain"]"""
os.remove(pyproject_path)
expected_docker_stdin = (
"""FROM langchain/langgraph-api:3.11
# -- Generate requirements.txt for packages without one --
# Copy packaging metadata files
ADD pyproject.toml /deps/unit_tests/pyproject.toml
# Generate requirements.txt from packaging metadata
# Compile from pyproject.toml for unit_tests
RUN cd '/deps/unit_tests' && uv pip compile pyproject.toml -o 'requirements.txt' --constraint /api/constraints.txt
# -- End of requirements.txt generation --
# -- Installing from requirements.txt files --
RUN PYTHONDONTWRITEBYTECODE=1 uv pip install --system --no-cache-dir -c /api/constraints.txt -r '/deps/unit_tests/requirements.txt'
# -- End of requirements.txt install --
# -- Adding local package . --
ADD . /deps/unit_tests
# -- End of local package . --
@@ -818,7 +836,7 @@ ENV LANGGRAPH_UI_CONFIG='{{"shared": ["nuqs"]}}'
ENV LANGSERVE_GRAPHS='{{"agent": "/deps/outer-unit_tests/unit_tests/agent.py:graph"}}'
# -- Installing JS dependencies --
ENV NODE_VERSION=20
RUN cd /deps/outer-unit_tests/unit_tests && npm i && tsx /api/langgraph_api/js/build.mts
RUN cd '/deps/outer-unit_tests/unit_tests' && npm i && tsx /api/langgraph_api/js/build.mts
# -- End of JS dependencies install --
{FORMATTED_CLEANUP_LINES}
WORKDIR /deps/outer-unit_tests/unit_tests"""
@@ -862,7 +880,7 @@ RUN for dep in /deps/*; do echo "Installing $dep"; if [
ENV LANGSERVE_GRAPHS='{{"python": "/deps/outer-unit_tests/unit_tests/multiplatform/python.py:graph", "js": "/deps/outer-unit_tests/unit_tests/multiplatform/js.mts:graph"}}'
# -- Installing JS dependencies --
ENV NODE_VERSION=22
RUN cd /deps/outer-unit_tests/unit_tests && npm i && tsx /api/langgraph_api/js/build.mts
RUN cd '/deps/outer-unit_tests/unit_tests' && npm i && tsx /api/langgraph_api/js/build.mts
# -- End of JS dependencies install --
{FORMATTED_CLEANUP_LINES}
WORKDIR /deps/outer-unit_tests/unit_tests"""