diff --git a/libs/cli/langgraph_cli/archive.py b/libs/cli/langgraph_cli/archive.py new file mode 100644 index 000000000..357214015 --- /dev/null +++ b/libs/cli/langgraph_cli/archive.py @@ -0,0 +1,133 @@ +"""Create a tarball of project source for remote builds.""" + +import os +import pathlib +import tarfile +import tempfile + +import click + + +_WARN_SIZE = 50 * 1024 * 1024 # 50 MB +_MAX_SIZE = 200 * 1024 * 1024 # 200 MB + + +def _tar_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: + """Strip symlinks, hardlinks, and traversal paths from archive.""" + if tarinfo.issym() or tarinfo.islnk(): + return None + if ".." in tarinfo.name.split("/"): + return None + return tarinfo + + +def _read_ignore_patterns(context_dir: pathlib.Path) -> list[str]: + """Read .dockerignore patterns if present.""" + dockerignore = context_dir / ".dockerignore" + if dockerignore.is_file(): + patterns = [] + for line in dockerignore.read_text().splitlines(): + line = line.strip() + if line and not line.startswith("#"): + patterns.append(line) + return patterns + return [] + + +def _should_ignore(rel_path: str, patterns: list[str]) -> bool: + """Check if a relative path matches any dockerignore pattern.""" + import fnmatch + + rel_path = rel_path.replace(os.sep, "/") + + for pattern in patterns: + negate = pattern.startswith("!") + if negate: + pattern = pattern[1:] + + if fnmatch.fnmatch(rel_path, pattern) or fnmatch.fnmatch( + rel_path, f"**/{pattern}" + ): + if negate: + return False + return True + + parts = rel_path.split("/") + for i in range(len(parts)): + partial = "/".join(parts[: i + 1]) + if fnmatch.fnmatch(partial, pattern): + if negate: + return False + return True + + return False + + +def create_archive( + config_path: pathlib.Path, +) -> tuple[str, int]: + """Create a .tar.gz archive of the project source. + + Returns (archive_path, file_size). + The archive root is config.parent (the directory containing langgraph.json). + """ + context_dir = config_path.parent.resolve() + config_filename = config_path.name + ignore_patterns = _read_ignore_patterns(context_dir) + + tmp_dir = tempfile.mkdtemp(prefix="langgraph-deploy-") + archive_path = os.path.join(tmp_dir, "source.tar.gz") + + with tarfile.open(archive_path, "w:gz") as tar: + for root, dirs, files in os.walk(context_dir): + rel_root = os.path.relpath(root, context_dir) + if rel_root == ".": + rel_root = "" + + dirs[:] = [ + d + for d in dirs + if not _should_ignore( + os.path.join(rel_root, d) if rel_root else d, ignore_patterns + ) + ] + + for f in files: + rel_path = os.path.join(rel_root, f) if rel_root else f + if _should_ignore(rel_path, ignore_patterns): + continue + full_path = os.path.join(root, f) + arcname = rel_path.replace(os.sep, "/") + info = tar.gettarinfo(full_path, arcname=arcname) + filtered = _tar_filter(info) + if filtered is None: + continue + with open(full_path, "rb") as fobj: + tar.addfile(filtered, fobj) + + file_size = os.path.getsize(archive_path) + + # Validate config file is at archive root + with tarfile.open(archive_path, "r:gz") as tar: + names = tar.getnames() + if config_filename not in names: + os.unlink(archive_path) + raise click.ClickException( + f"Archive validation failed: {config_filename} not found at archive root" + ) + + if file_size > _MAX_SIZE: + os.unlink(archive_path) + raise click.ClickException( + f"Source archive is {file_size / 1_048_576:.1f} MB, which exceeds the 200 MB limit. " + "Check your .dockerignore for large files (model weights, data, node_modules, .venv)." + ) + + if file_size > _WARN_SIZE: + click.secho( + f" Warning: source archive is {file_size / 1_048_576:.1f} MB. " + "Consider adding large files to .dockerignore.", + fg="yellow", + ) + + return archive_path, file_size diff --git a/libs/cli/langgraph_cli/cli.py b/libs/cli/langgraph_cli/cli.py index e8dd3cad2..66e0378e5 100644 --- a/libs/cli/langgraph_cli/cli.py +++ b/libs/cli/langgraph_cli/cli.py @@ -632,6 +632,14 @@ def build( @click.option("--install-command", hidden=True) @click.option("--build-command", hidden=True) @click.option("--api-version", type=str, hidden=True) +@click.option( + "--remote/--no-remote", + default=None, + help=( + "Force or disable remote build. Default: auto-detect " + "(use remote build when Docker is unavailable)." + ), +) @click.argument("docker_build_args", nargs=-1, type=click.UNPROCESSED) @cli.command( help=( @@ -659,6 +667,7 @@ def deploy( install_command: str | None, build_command: str | None, no_wait: bool, + remote: bool | None, docker_build_args: Sequence[str], ): config_json = langgraph_cli.config.validate_config_file(config) @@ -680,8 +689,76 @@ def deploy( secrets = _secrets_from_env(env_vars) - # Use buildx to cross-compile for amd64 when running on a non-x86_64 host - # (e.g. Apple Silicon). On amd64 hosts, plain docker build is sufficient. + # Determine whether to use remote build + use_remote = remote + if use_remote is None: + docker_available = langgraph_cli.docker.is_docker_available() + use_remote = not docker_available + elif use_remote is False: + pass # --no-remote: fail if Docker is missing (existing behavior) + + if use_remote: + _deploy_remote( + config=config, + config_json=config_json, + verbose=verbose, + api_version=api_version, + host_url=host_url, + api_key=api_key, + deployment_id=deployment_id, + deployment_type=deployment_type, + name=name, + base_image=base_image, + install_command=install_command, + build_command=build_command, + no_wait=no_wait, + secrets=secrets, + ) + else: + _deploy_local( + config=config, + config_json=config_json, + verbose=verbose, + api_version=api_version, + host_url=host_url, + api_key=api_key, + deployment_id=deployment_id, + deployment_type=deployment_type, + name=name, + image_name=image_name, + image_tag=image_tag, + base_image=base_image, + install_command=install_command, + build_command=build_command, + no_wait=no_wait, + pull=pull, + docker_build_args=docker_build_args, + secrets=secrets, + ) + + +def _deploy_local( + *, + config: pathlib.Path, + config_json: dict, + verbose: bool, + api_version: str | None, + host_url: str | None, + api_key: str, + deployment_id: str | None, + deployment_type: str, + name: str | None, + image_name: str | None, + image_tag: str, + base_image: str | None, + install_command: str | None, + build_command: str | None, + no_wait: bool, + pull: bool, + docker_build_args: Sequence[str], + secrets: list[dict[str, str]], +): + """Local Docker build + push deploy path.""" needs_buildx = platform.machine() != "x86_64" local_tag = f"langgraph-deploy-tmp:{int(time.time())}" @@ -697,7 +774,6 @@ def deploy( step = 1 - # -- Step: Build image -- log_step(f"{step}. Building image") if needs_buildx: build_flags: list[str] = [ @@ -742,48 +818,14 @@ def deploy( ) step += 1 - # -- Step: Find or create deployment -- client = HostBackendClient(host_url, api_key) - if deployment_id: - log_step(f"{step}. Using deployment {deployment_id}") - step += 1 - else: - log_step(f"{step}. Looking up deployment '{name}'") - existing = client.list_deployments(name_contains=name) - found_id = None - if isinstance(existing, dict): - for dep in existing.get("resources", []): - if isinstance(dep, dict) and dep.get("name") == name: - found_id = dep.get("id") - break - if found_id: - deployment_id = str(found_id) - click.secho( - f" Found existing deployment (ID: {deployment_id})", - fg="green", - ) - else: - log_step(f" Creating deployment '{name}'") - payload = { - "name": name, - "source": "internal_docker", - "source_config": {"deployment_type": deployment_type}, - "source_revision_config": {}, - "secrets": secrets, - } - created = client.create_deployment(payload) - created_id = created.get("id") if isinstance(created, dict) else None - if not isinstance(created_id, str) or not created_id: - raise HostBackendError( - "POST /v2/deployments succeeded but response " - "missing a valid 'id'" - ) - deployment_id = created_id - click.secho(f" Deployment ID: {deployment_id}", fg="green") - step += 1 + deployment_id = _find_or_create_deployment( + client, deployment_id, name, deployment_type, secrets, "internal_docker", + step, log_step, + ) + step += 1 - # -- Step: Get push token and authenticate -- log_step(f"{step}. Requesting push token") push_data = client.request_push_token(deployment_id) deployment_token = push_data.get("token") @@ -804,8 +846,6 @@ def deploy( registry_host = normalized_registry.split("/")[0] - # Use a clean Docker config with only the push token so that - # system credential helpers (e.g. gcloud) don't interfere. with _docker_config_for_token(registry_host, deployment_token) as cfg: log_step(f"{step}. Logging into {registry_host}") token_input = ( @@ -829,7 +869,6 @@ def deploy( ) step += 1 - # -- Step: Tag and push -- log_step(f"{step}. Pushing image {remote_image}") runner.run( subp_exec( @@ -853,7 +892,6 @@ def deploy( ) step += 1 - # -- Step: Update deployment -- log_step(f"{step}. Updating deployment {deployment_id}") client.update_deployment(deployment_id, remote_image, secrets=secrets) @@ -861,67 +899,317 @@ def deploy( click.secho(" Deployment updated", fg="green") return - # -- Poll revision status -- - revisions_resp = client.list_revisions(deployment_id, limit=1) - resources = ( - revisions_resp.get("resources", []) - if isinstance(revisions_resp, dict) - else [] - ) - if not resources: - click.secho(" Deployment updated", fg="green") - return + _poll_revision_status(client, deployment_id, verbose=verbose) - revision_id = str(resources[0]["id"]) - last_status = "" - deadline = time.time() + 300 - with Progress(message="Deploying...", elapsed=True) as set_progress: - while time.time() < deadline: - rev = client.get_revision(deployment_id, revision_id) - status = ( - rev.get("status", "UNKNOWN") if isinstance(rev, dict) else "UNKNOWN" +def _deploy_remote( + *, + config: pathlib.Path, + config_json: dict, + verbose: bool, + api_version: str | None, + host_url: str | None, + api_key: str, + deployment_id: str | None, + deployment_type: str, + name: str | None, + base_image: str | None, + install_command: str | None, + build_command: str | None, + no_wait: bool, + secrets: list[dict[str, str]], +): + """Remote build deploy path (no local Docker required).""" + import urllib.request + + from langgraph_cli.archive import create_archive + + def log_step(message: str) -> None: + click.secho(message, fg="cyan") + + step = 1 + click.secho("Docker not available. Using remote build.", fg="yellow") + + # -- Step: Create tarball -- + log_step(f"{step}. Creating source archive") + try: + archive_path, file_size = create_archive(config) + except KeyboardInterrupt: + click.echo("\nCancelled.") + raise click.exceptions.Exit(1) + click.secho( + f" Archive created ({file_size / 1_048_576:.1f} MB)", fg="green" + ) + step += 1 + + client = HostBackendClient(host_url, api_key) + + # -- Step: Find or create deployment -- + deployment_id = _find_or_create_deployment( + client, deployment_id, name, deployment_type, secrets, "internal_source", + step, log_step, + ) + step += 1 + + # -- Step: Request upload URL -- + log_step(f"{step}. Requesting upload URL") + upload_data = client.request_upload_url(deployment_id) + signed_url = upload_data.get("upload_url") + object_path = upload_data.get("object_path") + if not signed_url or not object_path: + raise click.ClickException("Upload URL response missing required fields") + step += 1 + + # -- Step: Upload tarball -- + log_step(f"{step}. Uploading source") + try: + _upload_to_gcs(signed_url, archive_path, file_size) + except KeyboardInterrupt: + click.echo("\nUpload cancelled.") + raise click.exceptions.Exit(1) + finally: + try: + os.unlink(archive_path) + except OSError: + pass + step += 1 + + # -- Step: Update deployment -- + log_step(f"{step}. Triggering remote build") + client.update_deployment_internal_source( + deployment_id, + source_tarball_path=object_path, + secrets=secrets, + config_path=config.name, + install_command=install_command, + build_command=build_command, + ) + step += 1 + + if no_wait: + click.secho(" Build triggered", fg="green") + return + + # -- Poll revision status with optional log streaming -- + _poll_revision_status(client, deployment_id, verbose=verbose, is_remote_build=True) + + +def _upload_to_gcs(signed_url: str, file_path: str, file_size: int) -> None: + """Upload tarball to GCS via signed PUT URL with progress display.""" + import urllib.request + + uploaded = 0 + + with open(file_path, "rb") as f: + original_read = f.read + + def tracked_read(size=-1): + nonlocal uploaded + data = original_read(size) + if data: + uploaded += len(data) + pct = int(uploaded * 100 / file_size) if file_size else 100 + click.echo( + f"\r Uploading ({file_size / 1_048_576:.1f} MB)... {pct}%", + nl=False, ) - if status != last_status: - last_status = status - set_progress("") - click.secho(f" Status: {status}", fg="cyan") - if status in _TERMINAL_STATUSES: - break - set_progress(f"{status}...") - time.sleep(1) - else: + return data + + f.read = tracked_read + + req = urllib.request.Request( + signed_url, + data=f, + method="PUT", + headers={ + "Content-Type": "application/gzip", + "Content-Length": str(file_size), + "X-Goog-Content-Length-Range": "0,209715200", + }, + ) + try: + urllib.request.urlopen(req) + except urllib.error.HTTPError as err: + detail = err.read().decode("utf-8", errors="ignore") + raise click.ClickException( + f"Upload failed with status {err.code}: {detail}" + ) from None + click.echo() + + +def _find_or_create_deployment( + client: HostBackendClient, + deployment_id: str | None, + name: str | None, + deployment_type: str, + secrets: list[dict[str, str]], + source: str, + step: int, + log_step: Callable[[str], None], +) -> str: + """Find an existing deployment or create a new one. Returns deployment_id.""" + if deployment_id: + log_step(f"{step}. Using deployment {deployment_id}") + return deployment_id + + log_step(f"{step}. Looking up deployment '{name}'") + existing = client.list_deployments(name_contains=name) + found_id = None + if isinstance(existing, dict): + for dep in existing.get("resources", []): + if isinstance(dep, dict) and dep.get("name") == name: + found_id = dep.get("id") + break + if found_id: + deployment_id = str(found_id) + click.secho( + f" Found existing deployment (ID: {deployment_id})", + fg="green", + ) + return deployment_id + + log_step(f" Creating deployment '{name}'") + payload = { + "name": name, + "source": source, + "source_config": {"deployment_type": deployment_type}, + "source_revision_config": {}, + "secrets": secrets, + } + created = client.create_deployment(payload) + created_id = created.get("id") if isinstance(created, dict) else None + if not isinstance(created_id, str) or not created_id: + raise HostBackendError( + "POST /v2/deployments succeeded but response missing a valid 'id'" + ) + deployment_id = created_id + click.secho(f" Deployment ID: {deployment_id}", fg="green") + return deployment_id + + +def _poll_revision_status( + client: HostBackendClient, + deployment_id: str, + *, + verbose: bool = False, + is_remote_build: bool = False, +) -> None: + """Poll revision status until terminal, optionally streaming build logs.""" + revisions_resp = client.list_revisions(deployment_id, limit=1) + resources = ( + revisions_resp.get("resources", []) + if isinstance(revisions_resp, dict) + else [] + ) + if not resources: + click.secho(" Deployment updated", fg="green") + return + + revision_id = str(resources[0]["id"]) + last_status = "" + log_offset: str | None = None + + deadline = time.time() + 900 if is_remote_build else time.time() + 300 + with Progress(message="Deploying...", elapsed=True) as set_progress: + while time.time() < deadline: + try: + rev = client.get_revision(deployment_id, revision_id) + except KeyboardInterrupt: set_progress("") + click.secho( + f"\n Interrupted. Deployment ID: {deployment_id}, " + f"Revision ID: {revision_id}", + fg="yellow", + ) + click.secho( + " The build will continue remotely.", + fg="yellow", + ) + raise click.exceptions.Exit(1) - dep_info = client.get_deployment(deployment_id) - custom_url = None - if isinstance(dep_info, dict): - sc = dep_info.get("source_config") - if isinstance(sc, dict): - custom_url = sc.get("custom_url") + status = ( + rev.get("status", "UNKNOWN") if isinstance(rev, dict) else "UNKNOWN" + ) + if status != last_status: + last_status = status + set_progress("") + click.secho(f" Status: {status}", fg="cyan") + if status in _TERMINAL_STATUSES: + break + set_progress(f"{status}...") - if last_status == "DEPLOYED": - click.secho(" Deployment successful!", fg="green") - if custom_url: - click.secho(f" URL: {custom_url}", fg="green") - elif last_status in ("BUILD_FAILED", "DEPLOY_FAILED", "CREATE_FAILED"): - click.secho(f" Deployment failed: {last_status}", fg="red") - raise click.exceptions.Exit(1) + # Stream build logs when verbose and building + if ( + is_remote_build + and verbose + and status in ("AWAITING_BUILD", "BUILDING") + ): + try: + logs_resp = client.list_build_logs( + deployment_id, revision_id, offset=log_offset + ) + if isinstance(logs_resp, dict): + for entry in logs_resp.get("logs", []): + msg = entry.get("message", "") + if msg: + click.echo(f" | {msg}") + log_offset = logs_resp.get("next_offset") or log_offset + except Exception: + pass + + time.sleep(3) else: + set_progress("") + + # On BUILD_FAILED, tail the last few build log lines + if is_remote_build and last_status == "BUILD_FAILED" and not verbose: + click.secho(" Last build log lines:", fg="red") + try: + logs_resp = client.list_build_logs( + deployment_id, revision_id, order="desc", limit=30 + ) + if isinstance(logs_resp, dict): + entries = list(reversed(logs_resp.get("logs", []))) + for entry in entries: + msg = entry.get("message", "") + if msg: + click.echo(f" | {msg}") + except Exception: + click.secho(" (failed to fetch build logs)", fg="red") + click.secho( + " Re-run with --verbose to see full build output.", + fg="yellow", + ) + + dep_info = client.get_deployment(deployment_id) + custom_url = None + if isinstance(dep_info, dict): + sc = dep_info.get("source_config") + if isinstance(sc, dict): + custom_url = sc.get("custom_url") + + if last_status == "DEPLOYED": + click.secho(" Deployment successful!", fg="green") + if custom_url: + click.secho(f" URL: {custom_url}", fg="green") + elif last_status in ("BUILD_FAILED", "DEPLOY_FAILED", "CREATE_FAILED"): + click.secho(f" Deployment failed: {last_status}", fg="red") + raise click.exceptions.Exit(1) + else: + click.secho( + f" Timed out waiting for deployment (last status: {last_status}).", + fg="yellow", + ) + if custom_url: click.secho( - f" Timed out waiting for deployment (last status: {last_status}).", + f" Check status at: {custom_url}", + fg="yellow", + ) + else: + click.secho( + " Check status in the LangSmith Deployments dashboard.", fg="yellow", ) - if custom_url: - click.secho( - f" Check status at: {custom_url}", - fg="yellow", - ) - else: - click.secho( - " Check status in the LangSmith Deployments dashboard.", - fg="yellow", - ) def _normalize_image_name(value: str | None) -> str: diff --git a/libs/cli/langgraph_cli/docker.py b/libs/cli/langgraph_cli/docker.py index 8dd02e0c7..feebb5de2 100644 --- a/libs/cli/langgraph_cli/docker.py +++ b/libs/cli/langgraph_cli/docker.py @@ -45,6 +45,23 @@ def _parse_version(version: str) -> Version: ) +def is_docker_available() -> bool: + """Check if Docker is installed and running without raising.""" + if shutil.which("docker") is None: + return False + try: + import subprocess + + result = subprocess.run( + ["docker", "info"], + capture_output=True, + timeout=10, + ) + return result.returncode == 0 + except Exception: + return False + + def check_capabilities( runner, *, require_compose: bool = True, require_buildx: bool = False ) -> DockerCapabilities: diff --git a/libs/cli/langgraph_cli/host_backend.py b/libs/cli/langgraph_cli/host_backend.py index c4a5cb423..a7fb64430 100644 --- a/libs/cli/langgraph_cli/host_backend.py +++ b/libs/cli/langgraph_cli/host_backend.py @@ -106,3 +106,58 @@ class HostBackendClient: "GET", f"/v2/deployments/{deployment_id}/revisions/{revision_id}", ) + + def request_upload_url(self, deployment_id: str) -> dict[str, Any]: + """Get a signed GCS URL for uploading the source tarball.""" + return self._request( + "POST", + f"/v2/deployments/{deployment_id}/upload-url", + ) + + def update_deployment_internal_source( + self, + deployment_id: str, + source_tarball_path: str, + secrets: list[dict[str, str]] | None = None, + config_path: str | None = None, + install_command: str | None = None, + build_command: str | None = None, + ) -> dict[str, Any]: + """Trigger a remote build revision with the uploaded tarball.""" + src_config: dict[str, Any] = { + "source_tarball_path": source_tarball_path, + } + if config_path is not None: + src_config["langgraph_config_path"] = config_path + + payload: dict[str, Any] = {"source_revision_config": src_config} + + source_config: dict[str, Any] = {} + if install_command is not None: + source_config["install_command"] = install_command + if build_command is not None: + source_config["build_command"] = build_command + if source_config: + payload["source_config"] = source_config + + if secrets is not None: + payload["secrets"] = secrets + return self._request("PATCH", f"/v2/deployments/{deployment_id}", payload) + + def list_build_logs( + self, + deployment_id: str, + revision_id: str, + order: str = "asc", + limit: int = 50, + offset: str | None = None, + ) -> dict[str, Any]: + """Fetch build logs for a revision.""" + payload: dict[str, Any] = {"order": order, "limit": limit} + if offset: + payload["offset"] = offset + return self._request( + "POST", + f"/v1/projects/{deployment_id}/revisions/{revision_id}/build_logs", + payload, + )