From e0abe06d33660e4ff5c9a20464feed75daa622f9 Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Wed, 3 Jun 2026 18:27:02 +0000 Subject: [PATCH 1/2] feat: container builds on release Signed-off-by: Matt Kornfield --- .dockerignore | 1 - .github/workflows/README.md | 11 +- .github/workflows/container-build.yml | 159 +++++++++++ .mise/tasks/container/build/gpu | 3 + .mise/tasks/container/build/gpu-dev | 3 + .mise/tasks/container/build/gpu-multiarch | 3 + STYLE_GUIDE.md | 2 +- containers/Dockerfile.cuda | 275 +++++++++---------- containers/README.md | 119 +++------ docs/developer-guide/docker.md | 308 +++++++++------------- docs/user-guide/docker.md | 13 +- 11 files changed, 478 insertions(+), 419 deletions(-) create mode 100644 .github/workflows/container-build.yml diff --git a/.dockerignore b/.dockerignore index 0f601dd6f..7980c6a1d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -60,7 +60,6 @@ DCO CITATION.md CODE_OF_CONDUCT.md SECURITY.md -THIRD_PARTY.md design.md # Dev-only dotfiles diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 829876868..afe810bc3 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -13,6 +13,7 @@ All workflows that use `.github/actions/setup-python-env` now default to the ver | -------------------------------------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------- | | [ci-checks.yml](ci-checks.yml) | Push to `main`, PRs, manual | Format, typecheck, unit tests, and CPU smoke tests | | [gpu-tests.yml](gpu-tests.yml) | Nightly, manual | GPU smoke tests (required) and E2E tests | +| [container-build.yml](container-build.yml) | Container/dependency PRs, `v*`, manual | Builds the extra-driven GPU container image and publishes GHCR tags for release tags and same-repo PRs | | [conventional-commit.yml](conventional-commit.yml) | PRs | Validates PR titles follow conventional commit format | | [docs.yml](docs.yml) | Push to `main` (docs paths) | Publishes `main` docs as the `latest` GitHub Pages version | | [release.yml](release.yml) | Push tags to `v*` | Builds and publishes package to Test PyPI/PyPI, creates a GitHub release, and publishes versioned docs | @@ -87,6 +88,11 @@ flowchart LR slackNotify[Slack Notification] end + subgraph containers [Container Build] + buildContainer[Build cu129 Image] + publishGhcr[Publish GHCR Tags] + end + subgraph internalRelease [Internal Release] buildWheelInt[Build Wheel] publishArtifactory[Publish to Artifactory/PyPI] @@ -95,10 +101,11 @@ flowchart LR push --> ci schedule --> gpu manual --> ci & gpu - pr --> ci & conventional & secrets - tag[Tag push v[0-9]*] --> release + pr --> ci & conventional & secrets & containers + tag[Tag push v[0-9]*] --> release & containers buildWheel --> publishPyPI --> ghRelease --> slackNotify + buildContainer --> publishGhcr buildWheelInt --> publishArtifactory conventional -.->|reuses| FW-CI-templates diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml new file mode 100644 index 000000000..2b07305eb --- /dev/null +++ b/.github/workflows/container-build.yml @@ -0,0 +1,159 @@ +# Copyright (c) 2024-2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Container Build + +on: + pull_request: + branches: + - main + paths: + - '.dockerignore' + - '.github/workflows/container-build.yml' + - 'containers/**' + - 'Makefile' + - 'pyproject.toml' + - 'uv.lock' + push: + tags: + - 'v*' + workflow_dispatch: + +defaults: + run: + shell: bash -x -e -u -o pipefail {0} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + packages: write + +env: + REGISTRY_IMAGE: ghcr.io/nvidia-nemo/safe-synthesizer + +jobs: + build: + name: Build ${{ matrix.variant }} image + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - variant: cu129 + extra: cu129 + platforms: linux/amd64 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Resolve package version + id: package-version + run: | + if [[ "${GITHUB_REF}" == refs/tags/v* ]]; then + version="${GITHUB_REF_NAME#v}" + else + version="0.0.0+${GITHUB_SHA::12}" + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + + - name: Log in to GHCR + if: >- + ${{ + github.event_name == 'push' || + ( + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + ) + }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + flavor: | + latest=false + tags: | + type=raw,value=${{ matrix.variant }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=raw,value=latest-${{ matrix.variant }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} + type=ref,event=pr,prefix=pr-,suffix=-${{ matrix.variant }} + type=sha,prefix=sha-,suffix=-${{ matrix.variant }} + type=semver,pattern={{version}}-${{ matrix.variant }} + type=semver,pattern={{major}}.{{minor}}-${{ matrix.variant }} + labels: | + org.opencontainers.image.version=${{ steps.package-version.outputs.version }} + com.nvidia.nemo.safe-synthesizer.extra=${{ matrix.extra }} + com.nvidia.nemo.safe-synthesizer.variant=${{ matrix.variant }} + + - name: Build and push image + if: >- + ${{ + github.event_name == 'push' || + ( + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + ) + }} + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + file: containers/Dockerfile.cuda + target: runtime + platforms: ${{ matrix.platforms }} + push: true + build-args: | + CONTAINER_EXTRA=${{ matrix.extra }} + CONTAINER_VARIANT=${{ matrix.variant }} + PACKAGE_VERSION=${{ steps.package-version.outputs.version }} + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache-${{ matrix.variant }} + cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache-${{ matrix.variant }},mode=max,oci-mediatypes=true,image-manifest=true + + - name: Build image + if: >- + ${{ + github.event_name != 'push' && + ( + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name != github.repository + ) + }} + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 + with: + context: . + file: containers/Dockerfile.cuda + target: runtime + platforms: ${{ matrix.platforms }} + push: false + build-args: | + CONTAINER_EXTRA=${{ matrix.extra }} + CONTAINER_VARIANT=${{ matrix.variant }} + PACKAGE_VERSION=${{ steps.package-version.outputs.version }} + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} diff --git a/.mise/tasks/container/build/gpu b/.mise/tasks/container/build/gpu index 3dc51eee5..1d3b05dbb 100755 --- a/.mise/tasks/container/build/gpu +++ b/.mise/tasks/container/build/gpu @@ -13,6 +13,9 @@ container_cmd="$(resolve_container_cmd)" --platform "${CONTAINER_GPU_PLATFORM:-linux/amd64}" \ --tag "${CONTAINER_GPU_IMAGE:-nss-gpu:latest}" \ --target runtime \ + --build-arg "CONTAINER_EXTRA=${CONTAINER_GPU_EXTRA:-cu129}" \ + --build-arg "CONTAINER_VARIANT=${CONTAINER_GPU_VARIANT:-${CONTAINER_GPU_EXTRA:-cu129}}" \ + --build-arg "PACKAGE_VERSION=${CONTAINER_GPU_PACKAGE_VERSION:-}" \ --progress=plain \ -f "${CONTAINER_GPU_FILE:-containers/Dockerfile.cuda}" \ . diff --git a/.mise/tasks/container/build/gpu-dev b/.mise/tasks/container/build/gpu-dev index 2c3e3c846..618746c3b 100755 --- a/.mise/tasks/container/build/gpu-dev +++ b/.mise/tasks/container/build/gpu-dev @@ -13,6 +13,9 @@ container_cmd="$(resolve_container_cmd)" --platform "${CONTAINER_GPU_PLATFORM:-linux/amd64}" \ --tag "${CONTAINER_GPU_IMAGE_DEV:-nss-gpu-dev:latest}" \ --target dev \ + --build-arg "CONTAINER_EXTRA=${CONTAINER_GPU_EXTRA:-cu129}" \ + --build-arg "CONTAINER_VARIANT=${CONTAINER_GPU_VARIANT:-${CONTAINER_GPU_EXTRA:-cu129}}" \ + --build-arg "PACKAGE_VERSION=${CONTAINER_GPU_PACKAGE_VERSION:-}" \ --progress=plain \ -f "${CONTAINER_GPU_FILE:-containers/Dockerfile.cuda}" \ . diff --git a/.mise/tasks/container/build/gpu-multiarch b/.mise/tasks/container/build/gpu-multiarch index 61a51f294..65cf4c118 100755 --- a/.mise/tasks/container/build/gpu-multiarch +++ b/.mise/tasks/container/build/gpu-multiarch @@ -25,6 +25,9 @@ docker buildx build \ --platform linux/amd64,linux/arm64 \ --tag "${CONTAINER_GPU_REGISTRY}/${CONTAINER_GPU_IMAGE:-nss-gpu:latest}" \ --target runtime \ + --build-arg "CONTAINER_EXTRA=${CONTAINER_GPU_EXTRA:-cu129}" \ + --build-arg "CONTAINER_VARIANT=${CONTAINER_GPU_VARIANT:-${CONTAINER_GPU_EXTRA:-cu129}}" \ + --build-arg "PACKAGE_VERSION=${CONTAINER_GPU_PACKAGE_VERSION:-}" \ --progress=plain \ --push \ -f "${CONTAINER_GPU_FILE:-containers/Dockerfile.cuda}" \ diff --git a/STYLE_GUIDE.md b/STYLE_GUIDE.md index c31b13d2a..8919bfeb6 100644 --- a/STYLE_GUIDE.md +++ b/STYLE_GUIDE.md @@ -765,7 +765,7 @@ Testing conventions are substantial enough to warrant their own section. For the Two Dockerfiles live in `containers/`: -- [containers/Dockerfile.cuda](containers/Dockerfile.cuda) -- CUDA GPU image (deps/runtime/dev stages). The production reference for these conventions. +- [containers/Dockerfile.cuda](containers/Dockerfile.cuda) -- CUDA GPU image (uv/runtime/dev stages). The production reference for these conventions. - [containers/Dockerfile.test_ci](containers/Dockerfile.test_ci) -- CPU-only CI image (`mise run test:ci-container`). See [containers/README.md](containers/README.md) for build arguments and mise tasks. diff --git a/containers/Dockerfile.cuda b/containers/Dockerfile.cuda index 42066c847..43a01aa0f 100644 --- a/containers/Dockerfile.cuda +++ b/containers/Dockerfile.cuda @@ -1,16 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Multistage CUDA Dockerfile for Safe-Synthesizer. +# Multistage GPU Dockerfile for Safe-Synthesizer. # -# Stages: -# tools -- install mise and all dev tools (uv, ruff, etc.) -# deps -- install Python, uv, and all cu129+engine dependencies -# runtime -- minimal CLI wrapper (ENTRYPOINT entrypoint.sh -> safe-synthesizer) -# dev -- extends runtime with dev tools, tests, and interactive shell +# CUDA support is selected through the project extras instead of a CUDA base +# image. The current distributable variant is cu129; adding cu130 later should +# only require a matching pyproject extra plus another workflow matrix row. # # Build: -# docker build -f containers/Dockerfile.cuda --target runtime -t nss-gpu:latest . +# docker build -f containers/Dockerfile.cuda \ +# --build-arg CONTAINER_EXTRA=cu129 \ +# --build-arg CONTAINER_VARIANT=cu129 \ +# --target runtime -t nss-gpu:latest . # docker build -f containers/Dockerfile.cuda --target dev -t nss-gpu-dev:latest . # # Run with your data: @@ -21,180 +22,141 @@ # nss-gpu:latest \ # run --data-source /workspace/data/input.csv # -# Key flags: -# --gpus all expose NVIDIA GPUs (requires nvidia-container-toolkit) -# --shm-size=1g increase /dev/shm for PyTorch training (default 64 MB is too small) -# -v HOST:CONTAINER bind-mount data and HF cache (use absolute paths) -# -e HF_HOME=... persist model downloads across container runs -# -e HF_TOKEN=... HF token for gated models (Llama, Mistral, etc.) -# -e NSS_INFERENCE_KEY=... inference API key for PII column classification (optional) -# -# Interactive shell: -# docker run -it --gpus all --shm-size=1g \ -# -v $(pwd)/my_data:/workspace/data \ -# -v ~/.cache/huggingface:/workspace/.hf_cache \ -# -e HF_HOME=/workspace/.hf_cache \ -# --entrypoint /bin/bash nss-gpu:latest -# -# Cache mounts (DOCKER_BUILDKIT=1, default in Docker 23.0+) avoid -# re-downloading ~10 GB of PyTorch/CUDA wheels on rebuild. -# # uv Docker best practices: https://docs.astral.sh/uv/guides/integration/docker/ -# --------------------------------------------------------------------------- -# Multi-architecture support -# --------------------------------------------------------------------------- -# This Dockerfile supports linux/amd64 and linux/arm64 (Blackwell/Grace). -# BuildKit sets TARGETARCH automatically when building with: -# docker buildx build --platform linux/arm64 ... -# -# The nvidia/cuda base images, ubuntu base images, and mise binaries are -# already multi-platform, so no conditional logic is needed for the core -# build. CUDA_ARCH_FLAGS is provided as forward-compatibility for when -# CUDA_IMAGE_TYPE=devel is used -# and kernels must be compiled for specific GPU architectures. -ARG TARGETARCH +ARG PYTHON_VERSION=3.13 +ARG PYTHON_IMAGE=python:${PYTHON_VERSION}-slim-bookworm +ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.9.30 # --------------------------------------------------------------------------- -# Build arguments +# Stage 1: uv -- copy the pinned uv binaries from the official image # --------------------------------------------------------------------------- -ARG CUDA_VERSION=12.9.1 -ARG UBUNTU_VERSION=22.04 -# Use "runtime" when all deps ship pre-built wheels (current state). -# Switch to "devel" if a future dependency requires CUDA compilation. -ARG CUDA_IMAGE_TYPE=runtime -ARG PYTHON_VERSION=3.13.13 - -# CUDA compute capabilities to compile for (used with CUDA_IMAGE_TYPE=devel). -# Recommended values per architecture: -# amd64 -> "80;86;90;90a" (A100, A10/3090, H100) -# arm64 -> "90;90a;120;120a" (H100 Grace, Blackwell) -# Not consumed by the current stages -- present for documentation and -# forward-compatibility with devel builds that invoke nvcc. -# Override via: --build-arg CUDA_ARCH_FLAGS="90;90a;120;120a" -ARG CUDA_ARCH_FLAGS="80;86;90;90a" +FROM ${UV_IMAGE} AS uv # --------------------------------------------------------------------------- -# Stage 1: tools -- install mise and all dev tools +# Stage 2: runtime -- slim image wrapping the safe-synthesizer CLI # --------------------------------------------------------------------------- -# Lightweight base (no CUDA needed for tool binaries). Mise is the single -# source of truth for tool versions via .mise.toml. -# See: https://mise.jdx.dev/mise-cookbook/docker.html -FROM ubuntu:${UBUNTU_VERSION} AS tools +FROM ${PYTHON_IMAGE} AS runtime -ARG UBUNTU_VERSION -ARG PYTHON_VERSION +ARG CONTAINER_EXTRA=cu129 +ARG CONTAINER_VARIANT=cu129 +ARG PACKAGE_VERSION= +ARG PYTHON_VERSION=3.13 +ARG OMIT_FLASHINFER_PACKAGES="--no-install-package flashinfer-cubin --no-install-package flashinfer-jit-cache" +ARG OMIT_TORCH_PACKAGES="--no-install-package torch --no-install-package torch-c-dlpack-ext --no-install-package torchaudio --no-install-package torchao --no-install-package torchvision --no-install-package triton" +ARG OMIT_VLLM_PACKAGES="--no-install-package vllm" -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ - apt-get update && apt-get install -y --no-install-recommends \ - git curl build-essential make ca-certificates gnupg gpg-agent dirmngr - -ENV MISE_DATA_DIR="/mise" \ - MISE_CONFIG_DIR="/mise" \ - MISE_CACHE_DIR="/mise/cache" \ - MISE_INSTALL_PATH="/usr/local/bin/mise" \ - PATH="/usr/local/bin:/mise/shims:${PATH}" - -WORKDIR /build -COPY .mise.toml mise.lock ./ -COPY .mise/tasks/ .mise/tasks/ -COPY tools/install-mise.sh ./tools/install-mise.sh -RUN MISE_GPG_KEY=24853EC9F655CE80B48E6C3A8B81C9D17413A06D \ - bash tools/install-mise.sh && \ - MISE_YES=1 mise trust && \ - PYTHON_VERSION="${PYTHON_VERSION}" MISE_YES=1 mise run setup - -# Stage real binaries to /usr/local/bin for stages that need the tool -# without the full mise runtime (shims require mise to resolve). -RUN cp "$(mise which uv)" /usr/local/bin/uv && \ - cp "$(mise which uvx)" /usr/local/bin/uvx - -# --------------------------------------------------------------------------- -# Stage 2: deps -- build dependencies in a CUDA environment -# --------------------------------------------------------------------------- -FROM nvidia/cuda:${CUDA_VERSION}-${CUDA_IMAGE_TYPE}-ubuntu${UBUNTU_VERSION} AS deps - -ARG PYTHON_VERSION +COPY --from=uv /uv /uvx /usr/local/bin/ -COPY --from=tools /usr/local/bin/uv /usr/local/bin/uv -COPY --from=tools /usr/local/bin/uvx /usr/local/bin/uvx - -# Cache apt downloads across rebuilds instead of deleting them each time. RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ apt-get update && apt-get install -y --no-install-recommends \ - git curl build-essential ca-certificates + bash \ + ca-certificates \ + libgomp1 \ + tini # uv settings for reproducible container builds. -# See: https://docs.astral.sh/uv/guides/integration/docker/#optimizations ENV UV_PROJECT_ENVIRONMENT=/opt/venv \ - UV_PYTHON_INSTALL_DIR=/opt/python \ - UV_PYTHON_CACHE_DIR=/root/.cache/uv/python \ + PATH="/opt/venv/bin:${PATH}" \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ UV_NO_INSTALLER_METADATA=1 \ - UV_FROZEN=true + UV_NO_MANAGED_PYTHON=1 \ + UV_FROZEN=true \ + UV_DYNAMIC_VERSIONING_BYPASS=${PACKAGE_VERSION} WORKDIR /build -# Install Python to a stable path (/opt/python) for cross-stage portability. -RUN --mount=type=cache,target=/root/.cache/uv \ - uv python install ${PYTHON_VERSION} - -# -- Intermediate layer: install dependencies without the project itself. -# This layer is invalidated only when pyproject.toml or uv.lock changes, -# not when source code changes. +# Install base dependencies first, then layer larger GPU dependency families. +# Each sync solves the final extra set but temporarily omits selected packages; +# later syncs install those omitted packages without removing prior layers. COPY pyproject.toml uv.lock ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --no-install-project --extra cu129 --extra engine --no-group dev + uv sync --python /usr/local/bin/python \ + --no-install-project \ + --no-group dev -# -- Project layer: install the project (non-editable so the venv is -# self-contained and source code is not needed in the runtime image). -COPY README.md ./ +# Engine dependencies are variant-neutral and can be reused by future CUDA +# extras. +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --python /usr/local/bin/python \ + --no-install-project \ + --extra engine \ + --no-group dev + +# Install the CUDA dependency closure before the largest binary package +# families. This captures NVIDIA runtime libraries separately from PyTorch, +# FlashInfer, and vLLM. +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --python /usr/local/bin/python \ + --no-install-project \ + --extra engine \ + --extra "${CONTAINER_EXTRA}" \ + ${OMIT_FLASHINFER_PACKAGES} \ + ${OMIT_TORCH_PACKAGES} \ + ${OMIT_VLLM_PACKAGES} \ + --no-group dev + +# Add FlashInfer binary/cache wheels in their own layer. +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --python /usr/local/bin/python \ + --no-install-project \ + --extra engine \ + --extra "${CONTAINER_EXTRA}" \ + ${OMIT_TORCH_PACKAGES} \ + ${OMIT_VLLM_PACKAGES} \ + --no-group dev + +# Add PyTorch, TorchVision, TorchAudio, TorchAO, and Triton separately from +# vLLM. +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --python /usr/local/bin/python \ + --no-install-project \ + --extra engine \ + --extra "${CONTAINER_EXTRA}" \ + ${OMIT_VLLM_PACKAGES} \ + --no-group dev + +# Install the remaining runtime dependencies, currently dominated by vLLM, +# before source files are copied. +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --python /usr/local/bin/python \ + --no-install-project \ + --extra engine \ + --extra "${CONTAINER_EXTRA}" \ + --no-group dev + +# Install the project non-editably so runtime does not need source code. +COPY LICENSE THIRD_PARTY.md README.md ./ COPY src/ src/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --no-editable --extra cu129 --extra engine --no-group dev - -# --------------------------------------------------------------------------- -# Stage 3: runtime -- minimal image wrapping the safe-synthesizer CLI -# --------------------------------------------------------------------------- -FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} AS runtime - -ARG CUDA_VERSION -ARG UBUNTU_VERSION - -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ - apt-get update && apt-get install -y --no-install-recommends \ - libgomp1 ca-certificates tini \ - gcc libc6-dev # torch.compile / Triton JIT needs a C compiler at runtime - -# Python toolchain and venv from the deps stage. -# UV_PYTHON_INSTALL_DIR=/opt/python keeps the path stable across stages. -COPY --from=deps /opt/python /opt/python -COPY --from=deps /opt/venv /opt/venv + uv sync --python /usr/local/bin/python \ + --no-editable \ + --extra engine \ + --extra "${CONTAINER_EXTRA}" \ + --no-group dev ENV PATH="/opt/venv/bin:${PATH}" \ + NSS_CONTAINER_EXTRA=${CONTAINER_EXTRA} \ + NSS_CONTAINER_VARIANT=${CONTAINER_VARIANT} \ NVIDIA_VISIBLE_DEVICES=all \ NVIDIA_DRIVER_CAPABILITIES=compute,utility -# OCI image metadata. LABEL org.opencontainers.image.title="Safe Synthesizer" \ - org.opencontainers.image.description="NVIDIA NeMo Safe Synthesizer -- GPU runtime for synthetic data generation, training, and evaluation." \ + org.opencontainers.image.description="NVIDIA NeMo Safe Synthesizer GPU runtime (${CONTAINER_VARIANT}); CUDA libraries are supplied by the ${CONTAINER_EXTRA} Python extra." \ org.opencontainers.image.vendor="NVIDIA" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.source="https://github.com/NVIDIA-NeMo/Safe-Synthesizer" \ org.opencontainers.image.documentation="https://nvidia-nemo.github.io/Safe-Synthesizer/user-guide/docker/" \ - org.opencontainers.image.base.name="nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}" \ + org.opencontainers.image.base.name="python:${PYTHON_VERSION}-slim-bookworm" \ + com.nvidia.nemo.safe-synthesizer.extra="${CONTAINER_EXTRA}" \ + com.nvidia.nemo.safe-synthesizer.variant="${CONTAINER_VARIANT}" \ org.opencontainers.image.usage="docker run --gpus all --shm-size=1g -v /path/to/data:/workspace/data -v ~/.cache/huggingface:/workspace/.hf_cache -e HF_HOME=/workspace/.hf_cache nss-gpu:latest run --config /workspace/data/config.yaml --data-source /workspace/data/input.csv" -# Non-root user. RUN groupadd -r -g 1000 appuser && \ useradd -r -u 1000 -g appuser -m appuser -# Wrapper entrypoint detects common misconfigurations (missing mounts, -# unset HF_HOME, no GPU) and prints hints before delegating to safe-synthesizer. COPY --chmod=755 containers/entrypoint.sh /usr/local/bin/entrypoint.sh USER appuser @@ -204,31 +166,34 @@ ENTRYPOINT ["tini", "--", "entrypoint.sh"] CMD ["--help"] # --------------------------------------------------------------------------- -# Stage 4: dev -- interactive development and testing (also CI test target) +# Stage 3: dev -- interactive development and testing # --------------------------------------------------------------------------- FROM runtime AS dev +ARG CONTAINER_EXTRA=cu129 +ARG PACKAGE_VERSION= + USER root +COPY --from=uv /uv /uvx /usr/local/bin/ + RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ apt-get update && apt-get install -y --no-install-recommends \ - git curl build-essential make gpg dirmngr gpg-agent - -# Mise toolchain from the tools stage -- no re-download needed. -COPY --from=tools /usr/local/bin/mise /usr/local/bin/mise -COPY --from=tools /mise /mise - -ENV MISE_DATA_DIR="/mise" \ - MISE_CONFIG_DIR="/mise" \ - MISE_CACHE_DIR="/mise/cache" \ - MISE_YES=1 \ - UV_PROJECT_ENVIRONMENT=/opt/venv \ - UV_PYTHON_INSTALL_DIR=/opt/python \ + bash \ + build-essential \ + ca-certificates \ + curl \ + git \ + make + +ENV UV_PROJECT_ENVIRONMENT=/opt/venv \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ + UV_NO_INSTALLER_METADATA=1 \ + UV_NO_MANAGED_PYTHON=1 \ UV_FROZEN=true \ - PATH="/mise/shims:${PATH}" + UV_DYNAMIC_VERSIONING_BYPASS=${PACKAGE_VERSION} WORKDIR /build @@ -236,9 +201,11 @@ WORKDIR /build # Bind-mount at /workspace overrides this at runtime for live editing. COPY . . -# Install dev dependencies into the existing venv. RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --extra cu129 --extra engine --group dev + uv sync --python /usr/local/bin/python \ + --extra "${CONTAINER_EXTRA}" \ + --extra engine \ + --group dev RUN git config --global --add safe.directory /workspace diff --git a/containers/README.md b/containers/README.md index 871f6f28a..c2f6c6233 100644 --- a/containers/README.md +++ b/containers/README.md @@ -9,17 +9,28 @@ Dockerfiles for running and testing Safe-Synthesizer in containers. | File | Base | Purpose | |------|------|---------| -| `Dockerfile.cuda` | `nvidia/cuda:12.8.1-runtime-ubuntu22.04` | GPU runtime and dev images for training, generation, and evaluation | +| `Dockerfile.cuda` | `python:3.13-slim-bookworm` | GPU runtime and dev images; CUDA libraries come from the selected package extra | | `Dockerfile.test_ci` | `python:3.13-slim` | CPU-only test image (`mise run test:ci-container`) | | `entrypoint.sh` | -- | Wrapper entrypoint for the runtime image (mount/GPU checks) | -## CUDA Image +## GPU Image -Three build stages, selected via `--target`: +`Dockerfile.cuda` is parameterized by the project extra: -- `runtime` -- minimal image wrapping the `safe-synthesizer` CLI via `tini` + `entrypoint.sh`. Runs as non-root `appuser`. The entrypoint detects common mistakes (empty workspace, missing HF cache, no HF token, no GPU, low `/dev/shm`) and prints hints before delegating to `safe-synthesizer`. -- `dev` -- extends runtime with git, uv, pytest, and the full dev dependency group. Runs as root for flexibility. -- `deps` -- intermediate stage (not a useful target on its own). +| Variant | Extra | Status | +|---------|-------|--------| +| `cu129` | `cu129` | Built today | +| `cu130` | `cu130` | Add after the CUDA 13.0 dependency extra lands | + +The image does not use `nvidia/cuda` as a base. PyTorch, vLLM, FlashInfer, +and the NVIDIA CUDA runtime libraries are installed from the locked Python +dependencies for the selected extra. + +Two image targets and one helper stage are available: + +- `uv` -- helper stage that provides pinned `uv` binaries from the official image. +- `runtime` -- slim CLI image that installs dependencies in layered sync steps, uses `tini` + `entrypoint.sh`, and runs as non-root `appuser`. The entrypoint detects common mistakes (empty workspace, missing HF cache, no HF token, no GPU, low `/dev/shm`) and prints hints before delegating to `safe-synthesizer`. +- `dev` -- extends runtime with `uv`, `make`, build tools, and the Python dev/test dependency group. ### Quick Start @@ -35,31 +46,6 @@ docker run --gpus all --shm-size=1g \ nss-gpu:latest \ run --data-source /workspace/data/input.csv - -# Run with a test dataset -docker run --gpus all --shm-size=1g \ - -v $(pwd)/tests/stub_datasets:/workspace/data \ - -v ~/.cache/huggingface:/workspace/.hf_cache \ - -e HF_HOME=/workspace/.hf_cache \ - nss-gpu:latest \ - run --data-source /workspace/data/clinc_oos.csv - -# Run the full pipeline with a config file -docker run --gpus all --shm-size=1g \ - -v $(pwd)/data:/workspace/data \ - -v ~/.cache/huggingface:/workspace/.hf_cache \ - -e HF_HOME=/workspace/.hf_cache \ - nss-gpu:latest \ - run --config /workspace/data/config.yaml --data-source /workspace/data/input.csv - -# Interactive shell (mount your data, override entrypoint) -docker run -it --gpus all --shm-size=1g \ - -v $(pwd)/data:/workspace/data \ - -v ~/.cache/huggingface:/workspace/.hf_cache \ - -e HF_HOME=/workspace/.hf_cache \ - --entrypoint /bin/bash \ - nss-gpu:latest - # Dev image with test tooling mise run container:build:gpu-dev CMD="mise run test" mise run container:run:gpu-dev @@ -68,32 +54,29 @@ CMD="mise run test" mise run container:run:gpu-dev Key flags: - `--gpus all` -- expose NVIDIA GPUs (requires nvidia-container-toolkit) -- `--shm-size=1g` -- increase `/dev/shm` for PyTorch training (default 64 MB causes "Bus error") -- `-v HOST:CONTAINER` -- bind-mount data and HF cache; Docker requires absolute paths (use `$(pwd)` to expand relative ones) +- `--shm-size=1g` -- increase `/dev/shm` for PyTorch training +- `-v HOST:CONTAINER` -- bind-mount data and HF cache; Docker requires absolute paths - `-e HF_HOME=...` -- persist model downloads across container runs -- `-e HF_TOKEN=...` -- Hugging Face token for gated models (Llama, Mistral, etc.) -- `-e NSS_INFERENCE_KEY=...` -- inference API key for PII column classification (optional; `NSS_INFERENCE_ENDPOINT` defaults to the NVIDIA integrate URL if unset) -- `-e WANDB_API_KEY=...` -- WandB API key for experiment tracking (optional) -- `--user "$(id -u):$(id -g)"` -- match host uid if you get "Permission denied" writing artifacts +- `-e HF_TOKEN=...` -- Hugging Face token for gated models +- `-e NSS_INFERENCE_KEY=...` -- inference API key for PII column classification ### Build Arguments | ARG | Default | Description | |-----|---------|-------------| -| `CUDA_VERSION` | `12.8.1` | CUDA toolkit version in the base image tag | -| `UBUNTU_VERSION` | `22.04` | Ubuntu version in the base image tag | -| `CUDA_IMAGE_TYPE` | `runtime` | Base image variant (`runtime` or `devel`) | -| `PYTHON_VERSION` | `3.13` | Python version installed via `uv python install` to `/opt/python` | -| `UV_VERSION` | `0.9.30` | uv version for the deps stage (matches `.mise.toml` pin) | -| `TARGETARCH` | _(set by BuildKit)_ | Target architecture (`amd64` or `arm64`) | -| `CUDA_ARCH_FLAGS` | `80;86;90;90a` | CUDA SM capabilities for `nvcc` (override for arm64: `90;90a;120;120a`) | +| `CONTAINER_EXTRA` | `cu129` | Python extra to install with `engine` | +| `CONTAINER_VARIANT` | `cu129` | Image variant label/tag suffix | +| `PACKAGE_VERSION` | unset | Optional PEP 440 version passed to `uv-dynamic-versioning` | +| `PYTHON_VERSION` | `3.13` | Python slim image version | +| `PYTHON_IMAGE` | `python:${PYTHON_VERSION}-slim-bookworm` | Runtime and dev base image | +| `UV_IMAGE` | `ghcr.io/astral-sh/uv:0.9.30` | Source image for pinned `uv` binaries | Override at build time: ```bash docker build -f containers/Dockerfile.cuda \ - --build-arg PYTHON_VERSION=3.12.10 \ - --build-arg CUDA_VERSION=12.6.3 \ + --build-arg CONTAINER_EXTRA=cu129 \ + --build-arg CONTAINER_VARIANT=cu129 \ --target runtime -t nss-gpu:custom . ``` @@ -109,47 +92,33 @@ docker build -f containers/Dockerfile.cuda \ See `mise tasks` for the full task list with usage hints. -## Multi-Architecture +Useful overrides: -The CUDA image supports `linux/amd64` and `linux/arm64` (Grace/Blackwell). +```bash +CONTAINER_GPU_EXTRA=cu129 \ +CONTAINER_GPU_VARIANT=cu129 \ +CONTAINER_GPU_IMAGE=nss-gpu-cu129:latest \ + mise run container:build:gpu +``` -### Single-arch builds +## Multi-Architecture -Override `CONTAINER_GPU_PLATFORM` to build for a specific architecture: +The Dockerfile accepts `--platform` through Docker/Buildx. The default local +target is `linux/amd64`: ```bash CONTAINER_GPU_PLATFORM=linux/arm64 mise run container:build:gpu ``` -### Multi-arch manifest - -Building a multi-platform manifest requires `docker buildx` and a registry -to push to -- `--load` only works for single-platform images: +Multi-platform manifests must be pushed to a registry: ```bash CONTAINER_GPU_REGISTRY=ghcr.io/nvidia-nemo mise run container:build:gpu-multiarch ``` -This pushes a manifest containing both `amd64` and `arm64` images to the -registry as `ghcr.io/nvidia-nemo/nss-gpu:latest`. - -### CUDA compute capabilities - -When `CUDA_IMAGE_TYPE=devel` is used and kernels must be compiled, set -`CUDA_ARCH_FLAGS` to the appropriate SM values: - -| Architecture | `CUDA_ARCH_FLAGS` | GPUs | -|--------------|-------------------|------| -| amd64 | `80;86;90;90a` | A100, A10/3090, H100 | -| arm64 | `90;90a;120;120a` | H100 Grace, Blackwell | - -```bash -docker build -f containers/Dockerfile.cuda \ - --build-arg CUDA_IMAGE_TYPE=devel \ - --build-arg CUDA_ARCH_FLAGS="90;90a;120;120a" \ - --platform linux/arm64 \ - --target runtime -t nss-gpu:arm64 . -``` +This builds and pushes `$(CONTAINER_GPU_REGISTRY)/$(CONTAINER_GPU_IMAGE)`. +Confirm the selected Python extra has wheels for every requested architecture +before enabling a platform in CI. ## CPU Test Image @@ -158,8 +127,6 @@ or in CI without a GPU. It uses a two-stage build: `setup` (system packages + mise-managed tools) and `install-deps` (Python environment via `mise run bootstrap-nss cpu`). -### Quick Start - ```bash # Run CI unit tests in a container mise run test:ci-container diff --git a/docs/developer-guide/docker.md b/docs/developer-guide/docker.md index a53c5db8d..d3edbd4be 100644 --- a/docs/developer-guide/docker.md +++ b/docs/developer-guide/docker.md @@ -3,8 +3,8 @@ # Docker: Build and Customize -How the CUDA Docker image is built, how to customize it, and how it relates -to the CI test image. +How the GPU Docker image is built, how variants map to dependency extras, and +how the image publication workflow is configured. For running Safe Synthesizer in a container, see [User Guide -- Docker](../user-guide/docker.md). @@ -14,83 +14,70 @@ For running Safe Synthesizer in a container, see ## Dockerfile Layout [`containers/Dockerfile.cuda`](https://github.com/NVIDIA-NeMo/Safe-Synthesizer/blob/main/containers/Dockerfile.cuda) -uses a four-stage multistage build: +uses `python:3.13-slim-bookworm` for the runtime and dev stages. CUDA support +comes from the selected Python extra rather than from an `nvidia/cuda` base +image. ```mermaid flowchart TD - ubuntuBase["ubuntu:UBUNTU_VERSION\n(tools stage)"] - cudaBuild["nvidia/cuda:CUDA_VERSION-CUDA_IMAGE_TYPE-ubuntu\n(deps stage -- may be runtime or devel)"] - cudaRuntime["nvidia/cuda:CUDA_VERSION-runtime-ubuntu\n(runtime stage -- always runtime)"] + uvImage["ghcr.io/astral-sh/uv:0.9.30\n(uv stage)"] + pythonBase["python:PYTHON_VERSION-slim-bookworm"] subgraph stages [Build Stages] - tools["tools\nInstalls mise + all dev tools\n(.mise.toml is single source of truth)"] - deps["deps\nInstalls Python 3.13 via uv\nuv sync cu129+engine"] - runtime["runtime\nCopies venv + Python\nNon-root appuser\ntini + entrypoint.sh"] - dev["dev\nExtends runtime\nCopies mise tree from tools\nRoot user"] + uv["uv\nCopies /uv and /uvx"] + runtime["runtime\nuv sync base deps\nuv sync engine+CONTAINER_EXTRA\nInstalls project into /opt/venv\nNon-root appuser\ntini + entrypoint.sh"] + dev["dev\nExtends runtime\nuv + make + dev/test deps\nRoot user"] end - ubuntuBase --> tools - tools -->|"COPY uv binary"| deps - cudaBuild --> deps - deps -->|"COPY venv + toolchain"| runtime - cudaRuntime --> runtime + uvImage --> uv + pythonBase --> runtime + uv -->|"COPY uv binaries"| runtime runtime --> dev - tools -->|"COPY mise + all tools"| dev + uv -->|"COPY uv binaries"| dev ``` -- tools: installs mise and all dev tools (uv, ruff, ty, gh, etc.) on a - lightweight `ubuntu` base. Mise is the single source of truth for tool - versions via `.mise.toml` -- no separate version pins in the Dockerfile. - Uses the [mise Docker cookbook](https://mise.jdx.dev/mise-cookbook/docker.html) - pattern with `MISE_DATA_DIR=/mise` for stable, copyable paths. -- deps: copies the uv binary from `tools`, then installs Python and all - cu129+engine dependencies. Uses `--mount=type=cache` to avoid - re-downloading ~10 GB of PyTorch/CUDA wheels. -- runtime: copies the venv and uv-managed Python into a fresh CUDA runtime - base. Runs as non-root `appuser` (uid 1000). GPU access is declared via - `NVIDIA_VISIBLE_DEVICES=all` and `NVIDIA_DRIVER_CAPABILITIES=compute,utility` - environment variables baked into the image. - Uses a wrapper entrypoint (`containers/entrypoint.sh`) that detects - common misconfigurations before delegating to `safe-synthesizer`. -- dev: extends runtime with the full mise toolchain (copied from `tools`) - and the dev dependency group (pytest, ruff, etc.). Runs as root for - flexibility. Used for interactive development and running tests inside - the container (also serves as the CI test target). +- `uv`: copies pinned `uv` binaries from the official image. +- `runtime`: installs base dependencies, then `engine` plus the selected + `CONTAINER_EXTRA`, then installs the project non-editably. These sync steps + run in the published image stage so the largest dependency families remain + separate pull layers. +- `dev`: adds `uv`, build tools, `make`, and the Python dev/test dependency + group so `make test` can run in-container. --- -## Entrypoint Script +## Variants -The runtime stage uses `containers/entrypoint.sh` instead of a bare -`ENTRYPOINT ["safe-synthesizer"]`. The script checks for common mistakes -and prints hints to stderr before calling `exec safe-synthesizer "$@"`: +The variant name is intentionally the same as the CUDA package extra. + +| Variant | Extra | Workflow status | +|---------|-------|-----------------| +| `cu129` | `cu129` | Enabled | +| `cu130` | `cu130` | Add when the CUDA 13.0 dependency PR lands | -- Empty `/workspace` -- user forgot to mount data -- `HF_HOME` not set or pointing to a nonexistent directory -- models will - download to a temporary location and be lost on exit -- `HF_TOKEN` missing and no cached token file -- gated models will fail -- `nvidia-smi` not found -- user may have forgotten `--gpus all` -- `/dev/shm` below 256 MB -- training with multi-worker data loading will - crash with "Bus error"; hints at `--shm-size=1g` +Adding a new variant should be mechanical: -These checks run only on stderr and do not interfere with normal CLI -output. For non-GPU commands (`--help`, `config validate`), the GPU check -is skipped. +1. Add the extra and source indexes to `pyproject.toml`. +2. Regenerate `uv.lock`. +3. Add a matrix row to `.github/workflows/container-build.yml`. +4. Build locally with `CONTAINER_GPU_EXTRA= CONTAINER_GPU_VARIANT=`. --- -## OCI Labels +## Entrypoint Script -The runtime image includes [OCI image metadata](https://github.com/opencontainers/image-spec/blob/main/annotations.md) -visible via `docker inspect`: +The runtime stage uses `containers/entrypoint.sh` instead of a bare +`ENTRYPOINT ["safe-synthesizer"]`. The script checks common mistakes and +prints hints to stderr before calling `exec safe-synthesizer "$@"`: -```bash -docker inspect nss-gpu:latest --format '{{index .Config.Labels "org.opencontainers.image.usage"}}' -``` +- Empty `/workspace` +- Missing or nonexistent `HF_HOME` +- Missing `HF_TOKEN` or cached Hugging Face token +- Missing `nvidia-smi` +- `/dev/shm` below 256 MB -Labels include `title`, `description`, `vendor`, `licenses`, `source`, -`documentation`, `base.name`, and `usage` (a full example `docker run` -command). +These checks do not interfere with normal CLI output. Info-only commands such +as `--help`, `--version`, and `config` skip runtime diagnostics. --- @@ -98,19 +85,20 @@ command). | ARG | Default | Description | |-----|---------|-------------| -| `CUDA_VERSION` | `12.8.1` | CUDA toolkit version in the base image tag | -| `UBUNTU_VERSION` | `22.04` | Ubuntu version in the base image tag | -| `CUDA_IMAGE_TYPE` | `runtime` | Base image variant for the deps stage. Change to `devel` if a dependency requires CUDA headers for compilation | -| `PYTHON_VERSION` | `3.13` | Python version installed via `uv python install` | -| `TARGETARCH` | _(set by BuildKit)_ | Target architecture (`amd64` or `arm64`). Automatically populated by `docker buildx build --platform` | -| `CUDA_ARCH_FLAGS` | `80;86;90;90a` | CUDA SM capabilities for `nvcc`. Override for arm64: `90;90a;120;120a` | +| `CONTAINER_EXTRA` | `cu129` | Python extra installed with `engine` | +| `CONTAINER_VARIANT` | `cu129` | Variant label/tag suffix | +| `PACKAGE_VERSION` | unset | Optional PEP 440 version passed via `UV_DYNAMIC_VERSIONING_BYPASS` | +| `PYTHON_VERSION` | `3.13` | Python slim image version | +| `PYTHON_IMAGE` | `python:${PYTHON_VERSION}-slim-bookworm` | Runtime/dev base image | +| `UV_IMAGE` | `ghcr.io/astral-sh/uv:0.9.30` | Source image for pinned `uv` binaries | Override at build time: ```bash docker build -f containers/Dockerfile.cuda \ - --build-arg PYTHON_VERSION=3.12.10 \ - --build-arg CUDA_VERSION=12.6.3 \ + --build-arg CONTAINER_EXTRA=cu129 \ + --build-arg CONTAINER_VARIANT=cu129 \ + --build-arg PACKAGE_VERSION=0.1.0 \ --target runtime -t nss-gpu:custom . ``` @@ -120,71 +108,45 @@ docker build -f containers/Dockerfile.cuda \ ### uv Environment Variables -The deps stage sets several uv environment variables for reproducible builds. -See the [uv Docker guide](https://docs.astral.sh/uv/guides/integration/docker/) -for full documentation. +The runtime and dev stages set: | Variable | Value | Why | |----------|-------|-----| | `UV_PROJECT_ENVIRONMENT` | `/opt/venv` | Installs into a fixed venv path | -| `UV_PYTHON_INSTALL_DIR` | `/opt/python` | Stable path for cross-stage COPY | -| `UV_PYTHON_CACHE_DIR` | `/root/.cache/uv/python` | Lets Python downloads benefit from the uv cache mount | -| `UV_LINK_MODE` | `copy` | Hardlinks into cache mounts vanish after unmount; copy is safe | -| `UV_COMPILE_BYTECODE` | `1` | Precompile `.pyc` for faster container startup | -| `UV_NO_INSTALLER_METADATA` | `1` | Deterministic layers (no `installer`/`direct_url.json` variance) | -| `UV_FROZEN` | `true` | Equivalent to `--frozen` on every uv command; prevents accidental re-locking | - -### NVIDIA Runtime Environment +| `UV_LINK_MODE` | `copy` | Cache-mount hardlinks do not survive outside the cache mount | +| `UV_COMPILE_BYTECODE` | `1` | Precompiles `.pyc` for faster startup | +| `UV_NO_INSTALLER_METADATA` | `1` | Reduces nondeterministic installer metadata | +| `UV_NO_MANAGED_PYTHON` | `1` | Forces use of the Python from the base image | +| `UV_FROZEN` | `true` | Prevents lockfile updates | +| `UV_DYNAMIC_VERSIONING_BYPASS` | `PACKAGE_VERSION` | Lets release workflows set package metadata without copying `.git` | -The runtime stage sets two environment variables that declare GPU -requirements to the NVIDIA Container Toolkit: - -| Variable | Value | Why | -|----------|-------|-----| -| `NVIDIA_VISIBLE_DEVICES` | `all` | Tells the toolkit this image needs GPU access. Equivalent to `--gpus all` intent; the user still passes `--gpus` to inject devices | -| `NVIDIA_DRIVER_CAPABILITIES` | `compute,utility` | Requests CUDA compute libraries and `nvidia-smi`. Matches the NMP convention (`nmp-gpu-base`) | - -This follows the same pattern used by NeMo Customizer and `nmp-gpu-base`. -No `video` group membership is needed -- the toolkit handles device -permissions when these variables are set. - -### Python Toolchain Portability - -`UV_PYTHON_INSTALL_DIR=/opt/python` puts the uv-managed Python at a -stable, explicit path instead of the default `~/.local/share/uv/python/`. -The venv at `/opt/venv` symlinks into that directory. Both paths are -copied to the runtime stage: - -```dockerfile -COPY --from=deps /opt/python /opt/python -COPY --from=deps /opt/venv /opt/venv -``` +### Runtime Dependency Layers -### Intermediate Layers +The runtime stage uses layered installation: -The deps stage uses two-pass installation following -[uv best practices](https://docs.astral.sh/uv/guides/integration/docker/#intermediate-layers): +1. `uv sync --no-install-project --no-group dev` installs the base package dependencies. +2. `uv sync --no-install-project --extra engine --no-group dev` installs engine dependencies. +3. `uv sync --no-install-project --extra engine --extra ${CONTAINER_EXTRA} --no-install-package ... --no-group dev` installs the CUDA dependency closure while omitting FlashInfer, PyTorch/Triton, and vLLM. +4. A second omitted-package sync adds FlashInfer binary/cache wheels. +5. A third omitted-package sync adds PyTorch, TorchVision, TorchAudio, TorchAO, and Triton while still omitting vLLM. +6. `uv sync --no-install-project --extra engine --extra ${CONTAINER_EXTRA} --no-group dev` installs the remaining runtime dependencies, currently dominated by vLLM. +7. `uv sync --no-editable --extra engine --extra ${CONTAINER_EXTRA} --no-group dev` installs Safe Synthesizer into the existing venv. -1. `uv sync --no-install-project` -- installs all dependencies from the - lockfile without the project itself. This layer is invalidated only when - `pyproject.toml` or `uv.lock` changes. -2. `uv sync --no-editable` -- installs the project non-editably into the - existing venv. Non-editable means the venv is self-contained and does - not need source code at runtime. +This keeps base and engine dependencies cached across CUDA extra changes, +splits the largest GPU dependency families into separate published image +layers, and keeps all dependency layers cached when only source files change. -### APT Cache Mounts +### NVIDIA Runtime Environment -APT layers use `--mount=type=cache,target=/var/cache/apt,sharing=locked` -instead of the traditional `rm -rf /var/lib/apt/lists/*` pattern. This -caches downloaded `.deb` files across rebuilds, speeding up layer -re-creation when the apt install list changes. +The runtime stage sets: -### Why runtime, Not devel +| Variable | Value | +|----------|-------| +| `NVIDIA_VISIBLE_DEVICES` | `all` | +| `NVIDIA_DRIVER_CAPABILITIES` | `compute,utility` | -All current locked dependencies (`torch`, `vllm`, `xformers`, `flashinfer`, -etc.) ship pre-built wheels. The CUDA devel image (~3 GB larger) is not -needed. If a future dependency requires source compilation with CUDA headers, -change `CUDA_IMAGE_TYPE` to `devel`. +The NVIDIA Container Toolkit injects host GPU devices, driver libraries, and +utility binaries such as `nvidia-smi` when the user runs with `--gpus all`. --- @@ -198,74 +160,79 @@ change `CUDA_IMAGE_TYPE` to `devel`. | `container:run:gpu` | Run a command in the runtime container | | `container:run:gpu-dev` | Run a command in the dev container | -For interactive shells, use `docker run -it --entrypoint /bin/bash` directly -- -this gives full control over mounts and flags. See the -[user guide](../user-guide/docker.md#interactive-shell) for examples. - Overridable variables: | Variable | Default | Description | |----------|---------|-------------| +| `CONTAINER_GPU_EXTRA` | `cu129` | Extra passed to `CONTAINER_EXTRA` | +| `CONTAINER_GPU_VARIANT` | `$(CONTAINER_GPU_EXTRA)` | Variant label passed to `CONTAINER_VARIANT` | +| `CONTAINER_GPU_PACKAGE_VERSION` | _(empty)_ | Version passed to `PACKAGE_VERSION` | | `CONTAINER_GPU_IMAGE` | `nss-gpu:latest` | Runtime image tag | | `CONTAINER_GPU_IMAGE_DEV` | `nss-gpu-dev:latest` | Dev image tag | -| `CONTAINER_GPU_PLATFORM` | `linux/amd64` | Target platform (override for arm64) | +| `CONTAINER_GPU_PLATFORM` | `linux/amd64` | Target platform | | `CONTAINER_GPU_REGISTRY` | _(empty)_ | Registry for multi-arch manifest pushes | | `CONTAINER_GPU_FLAG` | `--gpus all` | GPU access flag | | `CONTAINER_HF_CACHE` | `$(HOME)/.cache/huggingface` | Host HF cache dir | -| `CONTAINER_EXTRA_MOUNTS` | _(empty)_ | Additional `-v` flags for data outside the repo tree | +| `CONTAINER_EXTRA_MOUNTS` | _(empty)_ | Additional mounts for data outside the repo tree | --- -## Testing the Image +## Container Build Workflow -Smoke test (no GPU required): +`.github/workflows/container-build.yml` builds the runtime image on: -```bash -docker run --rm nss-gpu:latest --help -``` +- Pull requests that touch container or dependency inputs (`containers/**`, + `pyproject.toml`, `uv.lock`, `Makefile`, `.dockerignore`, or the workflow). +- Manual dispatch. +- Release tags. -Run unit tests inside the dev container: +Manual dispatch works for branch validation after this workflow exists on the +default branch. While adding the workflow for the first time, the path-filtered +pull request trigger is the pre-merge validation path. -```bash -CMD="mise run test" mise run container:run:gpu-dev -``` +The workflow pushes images for release tag `push` events and same-repository +pull requests. Fork pull requests build without pushing because their +`GITHUB_TOKEN` does not have package write permissions. -Interactive dev shell (use `docker run` directly for full mount control): +Build cache is exported to a dedicated GHCR registry cache tag, +`buildcache-`, only for events that can push packages. The workflow +does not use the GitHub Actions cache backend for Docker layers because the +CUDA dependency layers are large enough to churn the default Actions cache +quota and slow down cache export. -```bash -docker run -it --gpus all --shm-size=1g \ - -v $(pwd):/workspace \ - -v ~/.cache/huggingface:/workspace/.hf_cache \ - -e HF_HOME=/workspace/.hf_cache \ - --entrypoint /bin/bash \ - nss-gpu-dev:latest +Current image name: + +```text +ghcr.io/nvidia-nemo/safe-synthesizer ``` ---- +On release tags, current `cu129` tags include: -## Image Size +- `cu129` and `latest-cu129` +- `-cu129` and `.-cu129` on `v*` tags +- `sha--cu129` for traceability -The runtime image is approximately 15--25 GB, dominated by PyTorch, vllm, -and CUDA libraries. This is expected for a full ML inference stack. +On same-repository pull requests, current `cu129` tags include: -To reduce size: +- `pr--cu129` +- `sha--cu129` -- The runtime stage excludes dev dependencies (`--no-group dev`) -- The base is `cuda-runtime` (not `cuda-devel`) -- Build caches (`--mount=type=cache`) stay out of the final image layers +The workflow passes `PACKAGE_VERSION` into the Docker build. On release tags, +this is the tag without the leading `v`; on non-tag builds, it is +`0.0.0+`. --- ## Relationship to `Dockerfile.test_ci` -`Dockerfile.test_ci` provides a CPU-only test image for CI and local testing. +`Dockerfile.test_ci` provides a CPU-only test image for local CI checks. | Aspect | `Dockerfile.cuda` | `Dockerfile.test_ci` | |--------|-------------------|----------------------| -| Base | `nvidia/cuda:12.9.1-runtime-ubuntu22.04` | `python:3.13-slim` | -| Extras | `cu129` + `engine` | `cpu` + `engine` | -| GPU | Required | Not needed | -| Stages | `tools` / `deps` / `runtime` / `dev` | `setup` / `install-deps` | +| Base | `python:3.13-slim-bookworm` | `python:3.13-slim` | +| Extras | `CONTAINER_EXTRA` + `engine` | `cpu` + `engine` | +| GPU | Expected for runtime workloads | Not needed | +| Stages | `uv` / `runtime` / `dev` | `setup` / `install-deps` | | Use case | Training, generation, evaluation | CPU-only unit tests and CI checks | | Build task | `mise run container:build:gpu` | `mise run container:build:test` | @@ -282,9 +249,10 @@ Both follow the conventions in [STYLE_GUIDE.md -- Dockerfiles](https://github.co ## Multi-Architecture Support The CUDA Dockerfile supports `linux/amd64` and `linux/arm64` -(Grace/Blackwell). The `nvidia/cuda` base images, `ubuntu` base images, -and mise binaries are already multi-platform, so the same Dockerfile works -for both architectures without conditional logic. +(Grace/Blackwell) when the selected Python extra has compatible wheels for +the requested architecture. The Dockerfile relies on the Python slim base and +locked Python CUDA wheels, so there is no architecture-specific CUDA base +image selection in the Dockerfile. ### How it works @@ -296,9 +264,9 @@ docker buildx build --platform linux/arm64 \ -f containers/Dockerfile.cuda --target runtime -t nss-gpu:arm64 . ``` -No code paths branch on `TARGETARCH` today -- it exists as documentation -and forward-compatibility for when `CUDA_IMAGE_TYPE=devel` builds need to -pass architecture-specific flags to `nvcc`. +No code paths branch on `TARGETARCH` today; BuildKit selects the matching +base image architecture and the package resolver must find wheels compatible +with that platform. ### Building for arm64 (Blackwell) @@ -346,26 +314,6 @@ docker buildx inspect --bootstrap - For cross-architecture builds on amd64 hosts, QEMU user-static must be registered: `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes`. -### CUDA compute capabilities (`CUDA_ARCH_FLAGS`) - -When `CUDA_IMAGE_TYPE=devel` is used and CUDA kernels must be compiled, -pass the appropriate SM values via `--build-arg CUDA_ARCH_FLAGS`: - -| Architecture | `CUDA_ARCH_FLAGS` | GPUs | -|--------------|-------------------|------| -| amd64 | `80;86;90;90a` | A100, A10/3090, H100 | -| arm64 | `90;90a;120;120a` | H100 Grace, Blackwell | - -The Dockerfile defaults to the amd64 set. Override for arm64: - -```bash -docker build -f containers/Dockerfile.cuda \ - --build-arg CUDA_IMAGE_TYPE=devel \ - --build-arg CUDA_ARCH_FLAGS="90;90a;120;120a" \ - --platform linux/arm64 \ - --target runtime -t nss-gpu:arm64-devel . -``` - ### Mise tasks | Task | Description | diff --git a/docs/user-guide/docker.md b/docs/user-guide/docker.md index 65c14ff7a..b1868b813 100644 --- a/docs/user-guide/docker.md +++ b/docs/user-guide/docker.md @@ -13,13 +13,14 @@ for training, generation, and evaluation. - Docker 20.10+ (BuildKit enabled by default in 23.0+) - [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) installed and configured -- NVIDIA driver compatible with CUDA 12.9 +- NVIDIA driver compatible with the CUDA libraries installed by the image + variant (`cu129` today) - NVIDIA GPU (A100 or better recommended) Verify GPU access works: ```bash -docker run --rm --gpus all nvidia/cuda:12.8.1-runtime-ubuntu22.04 nvidia-smi +docker run --rm --gpus all nvidia/cuda:12.9.1-base-ubuntu22.04 nvidia-smi ``` --- @@ -258,12 +259,14 @@ mise run container:build:gpu # runtime image mise run container:build:gpu-dev # dev image with test tooling ``` -Override build arguments for different CUDA or Python versions: +Override build arguments for a different package extra, image variant, or +Python slim base version: ```bash docker build -f containers/Dockerfile.cuda \ - --build-arg CUDA_VERSION=12.6.3 \ - --build-arg PYTHON_VERSION=3.12.10 \ + --build-arg CONTAINER_EXTRA=cu129 \ + --build-arg CONTAINER_VARIANT=cu129 \ + --build-arg PYTHON_VERSION=3.12 \ --target runtime -t nss-gpu:custom . ``` From dfbe6824e3050b5be24f6ff7bc2bba4e967c669e Mon Sep 17 00:00:00 2001 From: Matt Kornfield Date: Mon, 8 Jun 2026 19:35:19 +0000 Subject: [PATCH 2/2] chore: remove PR builds Signed-off-by: Matt Kornfield --- .github/workflows/README.md | 4 +-- .github/workflows/container-build.yml | 38 +++------------------------ docs/developer-guide/docker.md | 22 +++++----------- 3 files changed, 11 insertions(+), 53 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index afe810bc3..c8f9c51a6 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -13,7 +13,7 @@ All workflows that use `.github/actions/setup-python-env` now default to the ver | -------------------------------------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------- | | [ci-checks.yml](ci-checks.yml) | Push to `main`, PRs, manual | Format, typecheck, unit tests, and CPU smoke tests | | [gpu-tests.yml](gpu-tests.yml) | Nightly, manual | GPU smoke tests (required) and E2E tests | -| [container-build.yml](container-build.yml) | Container/dependency PRs, `v*`, manual | Builds the extra-driven GPU container image and publishes GHCR tags for release tags and same-repo PRs | +| [container-build.yml](container-build.yml) | `v*`, manual | Builds the extra-driven GPU container image and publishes GHCR tags for release tags | | [conventional-commit.yml](conventional-commit.yml) | PRs | Validates PR titles follow conventional commit format | | [docs.yml](docs.yml) | Push to `main` (docs paths) | Publishes `main` docs as the `latest` GitHub Pages version | | [release.yml](release.yml) | Push tags to `v*` | Builds and publishes package to Test PyPI/PyPI, creates a GitHub release, and publishes versioned docs | @@ -101,7 +101,7 @@ flowchart LR push --> ci schedule --> gpu manual --> ci & gpu - pr --> ci & conventional & secrets & containers + pr --> ci & conventional & secrets tag[Tag push v[0-9]*] --> release & containers buildWheel --> publishPyPI --> ghRelease --> slackNotify diff --git a/.github/workflows/container-build.yml b/.github/workflows/container-build.yml index 2b07305eb..61354057c 100644 --- a/.github/workflows/container-build.yml +++ b/.github/workflows/container-build.yml @@ -15,16 +15,6 @@ name: Container Build on: - pull_request: - branches: - - main - paths: - - '.dockerignore' - - '.github/workflows/container-build.yml' - - 'containers/**' - - 'Makefile' - - 'pyproject.toml' - - 'uv.lock' push: tags: - 'v*' @@ -77,14 +67,7 @@ jobs: uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Log in to GHCR - if: >- - ${{ - github.event_name == 'push' || - ( - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository - ) - }} + if: github.event_name == 'push' uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 with: registry: ghcr.io @@ -101,7 +84,6 @@ jobs: tags: | type=raw,value=${{ matrix.variant }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} type=raw,value=latest-${{ matrix.variant }},enable=${{ startsWith(github.ref, 'refs/tags/v') }} - type=ref,event=pr,prefix=pr-,suffix=-${{ matrix.variant }} type=sha,prefix=sha-,suffix=-${{ matrix.variant }} type=semver,pattern={{version}}-${{ matrix.variant }} type=semver,pattern={{major}}.{{minor}}-${{ matrix.variant }} @@ -111,14 +93,7 @@ jobs: com.nvidia.nemo.safe-synthesizer.variant=${{ matrix.variant }} - name: Build and push image - if: >- - ${{ - github.event_name == 'push' || - ( - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository - ) - }} + if: github.event_name == 'push' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . @@ -136,14 +111,7 @@ jobs: cache-to: type=registry,ref=${{ env.REGISTRY_IMAGE }}:buildcache-${{ matrix.variant }},mode=max,oci-mediatypes=true,image-manifest=true - name: Build image - if: >- - ${{ - github.event_name != 'push' && - ( - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name != github.repository - ) - }} + if: github.event_name != 'push' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: . diff --git a/docs/developer-guide/docker.md b/docs/developer-guide/docker.md index d3edbd4be..1e2f19f4d 100644 --- a/docs/developer-guide/docker.md +++ b/docs/developer-guide/docker.md @@ -181,24 +181,19 @@ Overridable variables: `.github/workflows/container-build.yml` builds the runtime image on: -- Pull requests that touch container or dependency inputs (`containers/**`, - `pyproject.toml`, `uv.lock`, `Makefile`, `.dockerignore`, or the workflow). - Manual dispatch. - Release tags. Manual dispatch works for branch validation after this workflow exists on the -default branch. While adding the workflow for the first time, the path-filtered -pull request trigger is the pre-merge validation path. +default branch. Manual runs build the image without pushing it. -The workflow pushes images for release tag `push` events and same-repository -pull requests. Fork pull requests build without pushing because their -`GITHUB_TOKEN` does not have package write permissions. +The workflow pushes images only for release tag `push` events. Build cache is exported to a dedicated GHCR registry cache tag, -`buildcache-`, only for events that can push packages. The workflow -does not use the GitHub Actions cache backend for Docker layers because the -CUDA dependency layers are large enough to churn the default Actions cache -quota and slow down cache export. +`buildcache-`, only for release tag events that can push packages. The +workflow does not use the GitHub Actions cache backend for Docker layers +because the CUDA dependency layers are large enough to churn the default +Actions cache quota and slow down cache export. Current image name: @@ -212,11 +207,6 @@ On release tags, current `cu129` tags include: - `-cu129` and `.-cu129` on `v*` tags - `sha--cu129` for traceability -On same-repository pull requests, current `cu129` tags include: - -- `pr--cu129` -- `sha--cu129` - The workflow passes `PACKAGE_VERSION` into the Docker build. On release tags, this is the tag without the leading `v`; on non-tag builds, it is `0.0.0+`.