diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index cd81b1f..27040a7 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -54,7 +54,16 @@ therefore shadows the published one in your Docker daemon until you The build runs inside a hash-tagged **build-env image** (`env-`) based on `manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker cache, this repository's GHCR package, or a local build, reusing the cached -image on later runs. +image on later runs. It also accepts `--with-ca-certs`, which applies when the +image is built rather than found or pulled. + +`--with-ca-certs` (on either script) propagates the host's CA trust into the +build for corporate TLS gateways, using +[yscope-dev-utils' ca-trust library][ca-trust]. It's off by default and nothing +is baked into any image: the bundle is mounted only for the steps that need the +network, so CI needs no CA configuration at all. + +[ca-trust]: ../yscope-dev-utils/exports/docker/ca-trust/README.md Build state is cached under `.cache/` (`maven/`, `ccache/`, `fetchcontent//`, and `build//` for persisted CMake/build state), diff --git a/tools/build-packages/build-dependency-image.sh b/tools/build-packages/build-dependency-image.sh index e00d51b..5af588c 100755 --- a/tools/build-packages/build-dependency-image.sh +++ b/tools/build-packages/build-dependency-image.sh @@ -10,6 +10,12 @@ # docker run --rm -v "$(pwd):/src" -w /src "${image}" \ # task velox-connector:build-with-installed-deps # +# Options: +# --with-ca-certs Propagate the host's CA trust into the image build, for +# builds behind a corporate TLS gateway. Only applies when +# the image is built rather than found or pulled. Nothing is +# baked into the image. Off by default. +# # Requires: docker (with buildx), git, and sha256sum or shasum. set -o errexit @@ -29,6 +35,20 @@ host_platform() { main() { local build_env_hash image image_repo platform pull_err + local with_ca_certs=0 + + while [[ $# -gt 0 ]]; do + case $1 in + --with-ca-certs) + with_ca_certs=1 + shift + ;; + *) + echo >&2 "ERROR: unknown option: $1" + exit 1 + ;; + esac + done echo >&2 "==> Deriving build-env hash..." build_env_hash="$(derive_build_env_hash)" @@ -40,6 +60,9 @@ main() { echo >&2 " image: ${image}" if docker image inspect "${image}" &>/dev/null; then + if (( with_ca_certs )); then + echo >&2 " Note: --with-ca-certs applies only when the image is built; reusing cache." + fi echo >&2 "==> Found in local Docker cache." echo "${image}" return @@ -47,6 +70,9 @@ main() { echo >&2 "==> Checking repository registry..." if pull_err="$(docker pull "${image}" 2>&1)"; then + if (( with_ca_certs )); then + echo >&2 " Note: --with-ca-certs applies only when the image is built; pulled instead." + fi echo >&2 "==> Pulled from repository registry." echo "${image}" return @@ -56,9 +82,9 @@ main() { printf '%s\n' "${pull_err}" | sed 's/^/ /' >&2 echo >&2 "==> Image not available — building from scratch..." - build_image "${image}" "${platform}" "--load" + build_image "${image}" "${platform}" "--load" "${with_ca_certs}" echo >&2 "==> Built locally." echo "${image}" } -main +main "$@" diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index a1ce64c..0da1855 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -12,8 +12,6 @@ set -o pipefail src="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." &>/dev/null && pwd)" # shellcheck source=tools/build-packages/internal/build-cache/host.sh source "${src}/tools/build-packages/internal/build-cache/host.sh" -# shellcheck source=tools/build-packages/internal/ca-trust/host.sh -source "${src}/tools/build-packages/internal/ca-trust/host.sh" show_help() { cat <<'EOF' @@ -84,7 +82,11 @@ echo "==> Initializing submodules..." git -C "${src}" submodule update --init --recursive echo "==> Resolving build-env image..." -image=$("${src}/tools/build-packages/build-dependency-image.sh") +ca_certs_flag=() +if (( with_ca_certs )); then + ca_certs_flag=(--with-ca-certs) +fi +image=$("${src}/tools/build-packages/build-dependency-image.sh" ${ca_certs_flag[@]+"${ca_certs_flag[@]}"}) # FetchContent build state is compatible only with the image inputs identified # by this hash. image_hash="${image##*:env-}" @@ -109,19 +111,15 @@ prepare_build_cache "${src}/.cache" "${image_hash}" trust_mount_args=() if (( with_ca_certs )); then readonly TRUST_STAGE="${stage_dir}/trust" + # Sourced here rather than at the top of the script: the submodule is only + # guaranteed to exist after the `submodule update` above, and every + # invocation that doesn't use --with-ca-certs (including --help) must work + # on a fresh clone. + # shellcheck source=tools/yscope-dev-utils/exports/docker/ca-trust/host.sh + source "${src}/tools/yscope-dev-utils/exports/docker/ca-trust/host.sh" echo "==> Staging temporary container CA trust bundle..." - stage_host_ca_bundle "${TRUST_STAGE}" - if [[ ! -f "${TRUST_STAGE}/ca-bundle.pem" \ - || ! -r "${TRUST_STAGE}/ca-bundle.pem" \ - || ! -s "${TRUST_STAGE}/ca-bundle.pem" ]]; then - echo >&2 "ERROR: --with-ca-certs did not produce a usable host CA bundle" - exit 1 - fi - trust_mount_args=( - --mount "type=bind,src=${TRUST_STAGE},dst=${CA_TRUST_CONTAINER_DIR}" - --env "CA_TRUST_DIR=${CA_TRUST_CONTAINER_DIR}" - --env "CA_TRUST_JVM=1" - ) + ca_trust_stage_or_fail "${TRUST_STAGE}" + CA_TRUST_JVM=1 ca_trust_add_run_args trust_mount_args "${TRUST_STAGE}" fi host_uid=$(id -u) diff --git a/tools/build-packages/dependency-image/Dockerfile b/tools/build-packages/dependency-image/Dockerfile index 3feb42e..2f1ac54 100644 --- a/tools/build-packages/dependency-image/Dockerfile +++ b/tools/build-packages/dependency-image/Dockerfile @@ -1,39 +1,49 @@ # syntax=docker/dockerfile:1 -FROM quay.io/pypa/manylinux_2_28:latest +# Empty default for the `ca_trust` build context, so networked RUN steps below +# resolve their mount when no host CA trust is supplied. `build_image` overrides +# this stage with a staged directory when --with-ca-certs is passed; without it +# the mount is empty, the guard falls through, and the image keeps its own +# distro trust store. See yscope-dev-utils exports/docker/ca-trust. +FROM scratch AS ca_trust -ARG HOST_CA_BUNDLE=/run/secrets/host-ca -ARG HOST_CA_ENV=/usr/local/share/clp-plugin-presto-connector/use-host-ca.sh +FROM quay.io/pypa/manylinux_2_28:latest -# When available, expose the host CA bundle only to networked build steps via -# tool-specific environment variables. This supports local corporate builds -# without baking host CA certificates into the final image. -COPY tools/build-packages/dependency-image/use-host-ca.sh ${HOST_CA_ENV} +# NOTE: these duplicate the ca-trust library's CA_TRUST_CONTAINER_DIR and +# CA_TRUST_BUNDLE_FILENAME, which a Dockerfile can't read; keep them in sync. +# container.sh is sourced directly rather than via container-exec.sh because this +# base image's /bin/sh is bash. A base without that (Debian, Alpine) must switch to +# `bash /run/ca-trust/container-exec.sh `. +# Where the ca_trust context is mounted during networked RUN steps. Declared as +# an ARG so container.sh finds the staged bundle; nothing is baked into the +# image and the mount disappears with the step. +ARG CA_TRUST_DIR=/run/ca-trust +ARG CA_TRUST_BUNDLE=/run/ca-trust/ca-bundle.pem # Install OS packages missing from the manylinux base image. # dnf talks to libcurl directly and ignores CURL_CA_BUNDLE/SSL_CERT_FILE, so # it needs its own --setopt=sslcacert pointed at the host CA bundle. -RUN --mount=type=bind,from=host-ca,source=host-ca,target=${HOST_CA_BUNDLE} \ - . "${HOST_CA_ENV}" \ +RUN --mount=type=bind,from=ca_trust,target=/run/ca-trust \ + if [ -e /run/ca-trust/container.sh ]; then . /run/ca-trust/container.sh; fi \ && dnf_sslcacert_opt="" \ - && if [ -s "${HOST_CA_BUNDLE}" ]; then \ - dnf_sslcacert_opt="--setopt=sslcacert=${HOST_CA_BUNDLE}"; \ + && if [ -s "${CA_TRUST_BUNDLE}" ]; then \ + dnf_sslcacert_opt="--setopt=sslcacert=${CA_TRUST_BUNDLE}"; \ fi \ && dnf install -y --setopt=install_weak_deps=False ${dnf_sslcacert_opt} \ ccache dpkg gettext git java-17-openjdk-devel libcurl-devel libevent-devel \ libunwind-devel ninja-build openssl-devel patchelf python3-pip rpm-build \ && dnf clean all -RUN --mount=type=bind,from=host-ca,source=host-ca,target=${HOST_CA_BUNDLE} \ - . "${HOST_CA_ENV}" \ +RUN --mount=type=bind,from=ca_trust,target=/run/ca-trust \ + if [ -e /run/ca-trust/container.sh ]; then . /run/ca-trust/container.sh; fi \ && python3 -m pip install --no-cache-dir 'pip==21.3.1' \ && python3 -m pip install --no-cache-dir 'cmake==3.28.*' ARG GO_TASK_VERSION=v3.48.0 ARG TARGETARCH -RUN --mount=type=bind,from=host-ca,source=host-ca,target=${HOST_CA_BUNDLE} \ +RUN --mount=type=bind,from=ca_trust,target=/run/ca-trust \ set -euxo pipefail; \ - . "${HOST_CA_ENV}"; \ + if [ -e /run/ca-trust/container.sh ]; then . /run/ca-trust/container.sh; fi; \ mkdir -p /opt/go-task/bin; \ curl -fsSL "https://github.com/go-task/task/releases/download/${GO_TASK_VERSION}/task_linux_${TARGETARCH}.tar.gz" \ | tar xz -C /opt/go-task/bin task @@ -46,6 +56,6 @@ ENV TASK_TEMP_DIR=/opt/clp-plugin-presto-connector/.task # Build one dependency at a time to avoid excessive memory usage from parallel builds. RUN --mount=type=bind,source=.,target=/repo,readonly \ - --mount=type=bind,from=host-ca,source=host-ca,target=${HOST_CA_BUNDLE} \ - . "${HOST_CA_ENV}" \ + --mount=type=bind,from=ca_trust,target=/run/ca-trust \ + if [ -e /run/ca-trust/container.sh ]; then . /run/ca-trust/container.sh; fi \ && task --concurrency 1 -d /repo velox-connector:deps:install-all diff --git a/tools/build-packages/dependency-image/use-host-ca.sh b/tools/build-packages/dependency-image/use-host-ca.sh deleted file mode 100644 index 5cc9a74..0000000 --- a/tools/build-packages/dependency-image/use-host-ca.sh +++ /dev/null @@ -1,7 +0,0 @@ -if [ -s "${HOST_CA_BUNDLE}" ]; then - export CURL_CA_BUNDLE="${HOST_CA_BUNDLE}" - export GIT_SSL_CAINFO="${HOST_CA_BUNDLE}" - export PIP_CERT="${HOST_CA_BUNDLE}" - export REQUESTS_CA_BUNDLE="${HOST_CA_BUNDLE}" - export SSL_CERT_FILE="${HOST_CA_BUNDLE}" -fi diff --git a/tools/build-packages/dependency-image/utils.sh b/tools/build-packages/dependency-image/utils.sh index 08e3167..11848be 100644 --- a/tools/build-packages/dependency-image/utils.sh +++ b/tools/build-packages/dependency-image/utils.sh @@ -85,65 +85,56 @@ derive_build_env_hash() { ) } -# Stages the host CA bundle into the temporary Docker build context. -# -# Args: -_stage_host_ca_bundle() { - local dest="${1:?_stage_host_ca_bundle requires a destination path}" - local ca_bundle_candidates=( - "${SSL_CERT_FILE:-}" - /etc/ssl/certs/ca-certificates.crt - /etc/pki/tls/certs/ca-bundle.crt - /etc/ssl/cert.pem - ) - - local src - for src in "${ca_bundle_candidates[@]}"; do - [[ -f "${src}" && -s "${src}" ]] || continue - echo >&2 "==> Staging host CA bundle: ${src} -> ${dest}" - if ! cp "${src}" "${dest}"; then - echo >&2 "ERROR: failed to stage host CA bundle: ${src}" - return 1 - fi - return 0 - done - - echo >&2 "==> No host CA bundle found; continuing without host CA context." - return 1 -} - # ── Docker build ────────────────────────────────────────────────────────────── # Builds the dependency image. # # Args: -# $1 image tag — e.g. ghcr.io/owner/build-env:env- -# $2 platform — linux/amd64 or linux/arm64 -# $3 output flag — --push (registry) or --load (local docker) +# $1 image tag — e.g. ghcr.io/owner/build-env:env- +# $2 platform — linux/amd64 or linux/arm64 +# $3 output flag — --push (registry) or --load (local docker) +# $4 with CA certs — 1 to propagate the host's CA trust (optional, off by +# default). For local builds behind a corporate TLS +# gateway; CI has no such gateway and passes nothing, so +# the Dockerfile's empty `ca_trust` stage applies and the +# image keeps its own distro trust store. # # Requires: docker buildx, git build_image() { - local tag="$1" platform="$2" output="$3" + local tag="$1" platform="$2" output="$3" with_ca_certs="${4:-0}" - # Expose the host CA bundle as a narrow named build context so the Dockerfile - # can bind-mount it during networked RUN steps without baking it into image - # layers. Use a real context instead of a BuildKit secret because corporate CA - # bundles can exceed BuildKit's 500KiB secret limit. - local ca_stage; ca_stage=$(mktemp -d) + # stdout of this function is the caller's image ref; keep git chatter off it. + ensure_yscope_dev_utils_submodule >&2 + + local build_cmd=( + docker buildx build + --platform "${platform}" + --tag "${tag}" + "${output}" + -f "${_REPO_ROOT}/tools/build-packages/dependency-image/Dockerfile" + ) + + # String compare, not (( )): an arithmetic context name-resolves a non-numeric + # argument and aborts under `set -u`. + if [[ "${with_ca_certs}" != "1" ]]; then + build_cmd+=("${_REPO_ROOT}") + "${build_cmd[@]}" + return + fi + + # shellcheck source=tools/yscope-dev-utils/exports/docker/ca-trust/host.sh + source "${_REPO_ROOT}/tools/yscope-dev-utils/exports/docker/ca-trust/host.sh" + + local ca_stage + ca_stage="$(mktemp -d)" ( trap 'rm -rf "${ca_stage}"' EXIT - local ca_bundle="${ca_stage}/host-ca" - _stage_host_ca_bundle "${ca_bundle}" || : > "${ca_bundle}" - - ensure_yscope_dev_utils_submodule + ca_trust_stage_or_fail "${ca_stage}" + ca_trust_stage_build_context "${ca_stage}" + ca_trust_add_build_args build_cmd "${ca_stage}" - docker buildx build \ - --platform "${platform}" \ - --build-context "host-ca=${ca_stage}" \ - --tag "${tag}" \ - "${output}" \ - -f "${_REPO_ROOT}/tools/build-packages/dependency-image/Dockerfile" \ - "${_REPO_ROOT}" + build_cmd+=("${_REPO_ROOT}") + "${build_cmd[@]}" ) } diff --git a/tools/build-packages/internal/ca-trust/README.md b/tools/build-packages/internal/ca-trust/README.md deleted file mode 100644 index 9f382ed..0000000 --- a/tools/build-packages/internal/ca-trust/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# CA trust - -A reusable library for propagating the host's trusted certificates into a containerized build behind a corporate TLS gateway, without installing them in an image or persisting them in layers, caches, or artifacts. - -## Quick start - -On the host, stage the PEM CA bundle. Bind-mount it writable into the build container, and in the container set `CA_TRUST_DIR` to that mount point (plus `CA_TRUST_JVM=1` for a JVM build), then source `container.sh`. The Java PKCS#12 trust store is generated in-container into the same directory, alongside the bundle. - -```bash -# Host side -source tools/build-packages/internal/ca-trust/host.sh - -CA_TRUST_HOST_DIR="$(mktemp -d)" -trap 'rm -rf "${CA_TRUST_HOST_DIR}"' EXIT -stage_host_ca_bundle "${CA_TRUST_HOST_DIR}" # creates ${CA_TRUST_HOST_DIR}/ca-bundle.pem, read-only - -docker run --rm \ - --mount "type=bind,src=${CA_TRUST_HOST_DIR},dst=${CA_TRUST_CONTAINER_DIR}" \ - --env "CA_TRUST_DIR=${CA_TRUST_CONTAINER_DIR}" \ - --env "CA_TRUST_JVM=1" \ - --env MAVEN_OPTS \ - bash -c ' - source /repo/tools/build-packages/internal/ca-trust/container.sh - # ... run the build; curl/git/pip/Maven now use the host CAs - ' -``` - -Only the PEM bundle is staged on the host; the Java trust store is generated inside the container, which already has a JDK for the build. The generated store is written to the same writable bind mount (not the container's writable overlay), so it never lands on the overlay and cannot be retained by `docker commit`. The caller cleans up the staging directory. - -## Host API (`host.sh`) - -| Function | Args | Effect | -|------------------------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `stage_host_ca_bundle` | `` | Writes `/${CA_TRUST_BUNDLE_FILENAME}` (`0444`). Uses `SSL_CERT_FILE` when set, else searches common Linux CA-bundle locations; creates an empty file if none is found. | - -Constants: `CA_TRUST_BUNDLE_FILENAME` (`ca-bundle.pem`) and `CA_TRUST_CONTAINER_DIR` (`/run/ca-trust`, the in-container mount point for the staged trust directory, passed as `CA_TRUST_DIR`). - -## Container API (`container.sh`) - -Source it in the container after setting `CA_TRUST_DIR`; set `CA_TRUST_JVM=1` as well if the build runs on a JVM (Maven, Gradle, ...) that needs its trust store configured: - -```bash -CA_TRUST_DIR=/trusted -CA_TRUST_JVM=1 -source tools/build-packages/internal/ca-trust/container.sh -``` - -It reads `ca-bundle.pem` from `CA_TRUST_DIR`. When the bundle is non-empty it exports `CURL_CA_BUNDLE`, `GIT_SSL_CAINFO`, `PIP_CERT`, `REQUESTS_CA_BUNDLE`, and `SSL_CERT_FILE`. When `CA_TRUST_JVM` is set, the bundle is non-empty, and `keytool` is available, it also generates a PKCS#12 trust store from the bundle via `generators/java-pkcs12/generate.sh`, writes it to `${CA_TRUST_DIR}/truststore.p12`, and appends `-Djavax.net.ssl.trustStore*` to `MAVEN_OPTS` (preserving any caller-supplied value). - -**Persistence contract:** `CA_TRUST_DIR` must be a writable host bind-mount or tmpfs, not the container's writable overlay. `container.sh` verifies this with `findmnt` and refuses (with an error) to write to the overlay, since a file there would be retained by `docker commit`. If `findmnt` is unavailable it warns but proceeds. A generation failure errors. - -JVM trust configuration is opt-in via `CA_TRUST_JVM`, since not every caller runs on a JVM; it's also skipped when the bundle is empty or `keytool` is absent. A no-op when `CA_TRUST_DIR` is unset, so CI builds that don't mount a trust directory are unaffected. - -The caller owns and cleans up the staging directory; the scripts never modify the host or container trust stores, only the staged bundle. The generated PKCS#12 store is a per-build file in the caller's staging directory, removed when the caller cleans up. - -## Extensibility - -Add a backend under `generators/` when a trust format can't consume the PEM bundle directly. Keep host discovery and lifecycle in `host.sh`; keep format-specific conversion in the backend, run in-container. See `generators/java-pkcs12/README.md`. diff --git a/tools/build-packages/internal/ca-trust/container.sh b/tools/build-packages/internal/ca-trust/container.sh deleted file mode 100644 index 9e059ce..0000000 --- a/tools/build-packages/internal/ca-trust/container.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash - -# Container-side configuration for CA trust. Source it after setting -# CA_TRUST_DIR to a writable mount of the staged trust directory, which must -# contain ca-bundle.pem. Set CA_TRUST_JVM=1 as well if the build runs on a JVM -# (Maven, Gradle, ...) that needs its trust store configured: a Java PKCS#12 -# trust store is then generated here, inside the container, from the PEM -# bundle using the container's own JDK (keytool) -- no separate generator -# container or host JDK is required -- and written back to CA_TRUST_DIR -# alongside the bundle. -# -# Persistence contract: CA_TRUST_DIR must be a writable host bind-mount (or -# tmpfs), not the container's writable overlay. A file on the overlay is retained -# by `docker commit`; a bind mount is not part of any committed image. This -# script refuses to write to the overlay. - -if [[ -z "${CA_TRUST_DIR:-}" ]]; then - return 0 2>/dev/null || exit 0 -fi - -_ca_trust_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -HOST_CA_BUNDLE="${CA_TRUST_DIR}/ca-bundle.pem" - -if [[ -s "${HOST_CA_BUNDLE:-}" ]]; then - export CURL_CA_BUNDLE="${HOST_CA_BUNDLE}" - export GIT_SSL_CAINFO="${HOST_CA_BUNDLE}" - export PIP_CERT="${HOST_CA_BUNDLE}" - export REQUESTS_CA_BUNDLE="${HOST_CA_BUNDLE}" - export SSL_CERT_FILE="${HOST_CA_BUNDLE}" -fi - -# Generate a Java PKCS#12 trust store in-container from the staged PEM bundle and -# point Maven at it. Opt-in via CA_TRUST_JVM=1, since not every caller of this -# library runs on a JVM. Also skipped when the bundle is empty or keytool is -# unavailable, so CI builds without a trust directory and PEM-only staging -# (empty bundle) are unaffected. -if [[ -n "${CA_TRUST_JVM:-}" ]] && [[ -s "${HOST_CA_BUNDLE:-}" ]] && command -v keytool &>/dev/null; then - if ! mkdir -p "${CA_TRUST_DIR}"; then - echo >&2 "ERROR: cannot create Java trust store dir: ${CA_TRUST_DIR}" - return 1 2>/dev/null || exit 1 - fi - - # Refuse to write to the container's writable overlay: a file there is - # retained by `docker commit`, violating the no-persistence invariant. A - # bind mount or tmpfs has its own mount target; the root overlay resolves - # to "/". Warn (but proceed) if findmnt is unavailable to check. - if command -v findmnt &>/dev/null; then - _ca_trust_mount_target="$(findmnt -T "${CA_TRUST_DIR}" -o TARGET -n 2>/dev/null || true)" - if [[ -z "${_ca_trust_mount_target}" || "${_ca_trust_mount_target}" == "/" ]]; then - echo >&2 "ERROR: CA_TRUST_DIR (${CA_TRUST_DIR}) is on the container's writable overlay," - echo >&2 " which docker commit would retain. Mount a writable host directory or tmpfs there." - return 1 2>/dev/null || exit 1 - fi - else - echo >&2 "WARNING: findmnt unavailable; cannot verify CA_TRUST_DIR is off the overlay." - fi - - HOST_CA_JAVA_TRUST_STORE="${CA_TRUST_DIR}/truststore.p12" - if ! bash "${_ca_trust_dir}/generators/java-pkcs12/generate.sh" \ - "${HOST_CA_BUNDLE}" "${HOST_CA_JAVA_TRUST_STORE}"; then - echo >&2 "ERROR: failed to generate Java PKCS#12 trust store from ${HOST_CA_BUNDLE}" - return 1 2>/dev/null || exit 1 - fi - - # Preserve any Maven options supplied by the caller. - _host_ca_maven_opts="${MAVEN_OPTS:-}" - [[ -n "${_host_ca_maven_opts}" ]] && _host_ca_maven_opts="${_host_ca_maven_opts} " - _host_ca_maven_opts="${_host_ca_maven_opts}-Djavax.net.ssl.trustStore=${HOST_CA_JAVA_TRUST_STORE}" - _host_ca_maven_opts="${_host_ca_maven_opts} -Djavax.net.ssl.trustStoreType=PKCS12" - # The store contains only public certificates; this is an integrity password, not a secret. - _host_ca_maven_opts="${_host_ca_maven_opts} -Djavax.net.ssl.trustStorePassword=changeit" - export MAVEN_OPTS="${_host_ca_maven_opts}" - unset _host_ca_maven_opts _ca_trust_mount_target -fi - -unset _ca_trust_dir diff --git a/tools/build-packages/internal/ca-trust/generators/java-pkcs12/README.md b/tools/build-packages/internal/ca-trust/generators/java-pkcs12/README.md deleted file mode 100644 index 8e93ab7..0000000 --- a/tools/build-packages/internal/ca-trust/generators/java-pkcs12/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Java PKCS#12 generator - -A `generators/` backend that produces a Java PKCS#12 trust store from a PEM CA -bundle. Invoked by `container.sh` inside the build container; also runnable -directly. - -## Usage - -```bash -./tools/build-packages/internal/ca-trust/generators/java-pkcs12/generate.sh \ - -``` - -It needs a JDK: it locates `keytool` via `JAVA_HOME`, falling back to `keytool` -on `PATH`, then reads the JDK's base trust store (`jssecacerts` if present, else -`cacerts`). Given the inputs, it: - -1. Copies the base JDK trust store into a new PKCS#12 store via - `keytool -importkeystore`, preserving the standard Mozilla CA set alongside - the host's corporate CAs. -2. Imports each certificate from the PEM bundle with `keytool -importcert`, - splitting the bundle first (keytool reads only the first certificate from a - multi-cert PEM file) and using unique `host-ca-` aliases. Certificates - already present under any alias are silently skipped. -3. Writes the result to the output path (store password `changeit`, an - integrity password for public certificates, not a secret). - -`container.sh` runs this and feeds the result to Maven via -`-Djavax.net.ssl.trustStore= -Djavax.net.ssl.trustStoreType=PKCS12 --Djavax.net.ssl.trustStorePassword=changeit`, appended to `MAVEN_OPTS`, avoiding -edits to the JDK's installed `cacerts`. - -## Notes - -The generator runs in the build container, which already has a JDK for the -build, so no separate generator container or host JDK is required. The output -store is written to the caller-supplied output path, which `container.sh` places -in `CA_TRUST_DIR` -- a writable host bind-mount, not the container's writable -overlay -- so it never enters the image, caches, packages, or layers and is -cleaned up by the caller. - -## Files - -- `generate.sh` -- validates inputs, locates the JDK trust store, runs keytool. \ No newline at end of file diff --git a/tools/build-packages/internal/ca-trust/generators/java-pkcs12/generate.sh b/tools/build-packages/internal/ca-trust/generators/java-pkcs12/generate.sh deleted file mode 100755 index 0012389..0000000 --- a/tools/build-packages/internal/ca-trust/generators/java-pkcs12/generate.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash - -# Generates a Java PKCS#12 trust store from a PEM CA bundle, merging the -# selected JDK's default certificates with the bundle's certificates. -# -# Runs inside the build container, which already provides a JDK (keytool + -# cacerts); no separate generator container or host JDK is required. - -set -o errexit -set -o nounset -set -o pipefail - -if (( $# != 2 )) || [[ -z "$1" || -z "$2" ]]; then - echo >&2 "ERROR: generate.sh requires an input CA bundle and output path" - exit 2 -fi - -input_bundle="$1" -output_trust_store="$2" -if [[ ! -f "${input_bundle}" || ! -r "${input_bundle}" ]]; then - echo >&2 "ERROR: input CA bundle is not a readable regular file: ${input_bundle}" - exit 1 -fi -output_dir="$(dirname "${output_trust_store}")" -if [[ ! -d "${output_dir}" || ! -w "${output_dir}" ]]; then - echo >&2 "ERROR: output directory is not writable: ${output_dir}" - exit 1 -fi -if [[ -e "${output_trust_store}" && ! -f "${output_trust_store}" ]]; then - echo >&2 "ERROR: output path is not a regular file: ${output_trust_store}" - exit 1 -fi - -# Integrity password for a store of public CA certificates; not a secret. -readonly STOREPASS=changeit - -# Locate keytool and the JDK's default trust store. Match Java's trust-store -# lookup order: jssecacerts overrides cacerts. -java_home="${JAVA_HOME:-}" -if [[ -n "${java_home}" ]]; then - keytool="${java_home}/bin/keytool" -else - keytool="$(command -v keytool)" || { - echo >&2 "ERROR: keytool was not found in PATH and JAVA_HOME is unset" - exit 1 - } - keytool="$(readlink -f "${keytool}")" - java_home="${keytool%/bin/keytool}" -fi -if [[ ! -x "${keytool}" ]]; then - echo >&2 "ERROR: keytool is not executable: ${keytool}" - exit 1 -fi - -java_security_dir="${java_home}/lib/security" -base_java_trust_store="${java_security_dir}/cacerts" -if [[ -f "${java_security_dir}/jssecacerts" && -s "${java_security_dir}/jssecacerts" ]]; then - base_java_trust_store="${java_security_dir}/jssecacerts" -fi -if [[ ! -f "${base_java_trust_store}" || ! -r "${base_java_trust_store}" \ - || ! -s "${base_java_trust_store}" ]]; then - echo >&2 "ERROR: JDK default trust store is not readable: ${base_java_trust_store}" - exit 1 -fi - -# Append each certificate from the PEM bundle. keytool -importcert reads only -# the first certificate from a multi-cert PEM file, so split the bundle into -# per-cert buffers and import each under a unique alias. -work_dir="$(mktemp -d)" -trap 'rm -rf "${work_dir}"' EXIT - -# Start from a copy of the JDK's default trust store as PKCS#12. This keeps the -# standard Mozilla CA set alongside the host's corporate CAs, so downloads to -# public mirrors (not behind the corporate gateway) still verify. keytool prints -# one progress line per entry to stderr; capture it so success is quiet but a -# failure still surfaces the cause. -if ! "${keytool}" -importkeystore -noprompt \ - -srckeystore "${base_java_trust_store}" -srcstoretype JKS -srcstorepass "${STOREPASS}" \ - -destkeystore "${output_trust_store}" -deststoretype PKCS12 -deststorepass "${STOREPASS}" \ - >/dev/null 2>"${work_dir}/import.err"; then - echo >&2 "ERROR: keytool -importkeystore failed:" - cat >&2 "${work_dir}/import.err" - exit 1 -fi - -count=0 -cert_buf="" -cert_file="${work_dir}/cert.pem" -while IFS= read -r line || [[ -n "${line}" ]]; do - cert_buf+="${line}"$'\n' - if [[ "${line}" == "-----END CERTIFICATE-----" ]]; then - printf '%s' "${cert_buf}" > "${cert_file}" - # -noprompt skips the "trust this certificate?" prompt. A certificate - # already present under any alias is silently skipped by keytool, so - # duplicates in the bundle (or shared with cacerts) are harmless. - if ! "${keytool}" -importcert -noprompt \ - -alias "host-ca-${count}" -file "${cert_file}" \ - -keystore "${output_trust_store}" -storetype PKCS12 -storepass "${STOREPASS}" \ - >/dev/null 2>"${work_dir}/import-cert.err"; then - echo >&2 "ERROR: failed to import certificate #${count} from bundle:" - cat >&2 "${work_dir}/import-cert.err" - exit 1 - fi - count=$((count + 1)) - cert_buf="" - fi -done < "${input_bundle}" - -if (( count == 0 )); then - echo >&2 "ERROR: input bundle contains no complete PEM certificates: ${input_bundle}" - exit 1 -fi - -if [[ ! -s "${output_trust_store}" ]]; then - echo >&2 "ERROR: generated trust store is empty: ${output_trust_store}" - exit 1 -fi -echo "==> Generated Java PKCS#12 trust store: ${output_trust_store} (${count} bundle certificate(s) processed)" diff --git a/tools/build-packages/internal/ca-trust/host.sh b/tools/build-packages/internal/ca-trust/host.sh deleted file mode 100644 index c52e5f8..0000000 --- a/tools/build-packages/internal/ca-trust/host.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash - -# Host-side CA discovery and staging shared by Docker build and run workflows. - -if [[ "${_CA_TRUST_HOST_SH_LOADED:-}" == "1" ]]; then - return 0 -fi -readonly _CA_TRUST_HOST_SH_LOADED=1 - -# Conventional staged filename for the host CA bundle. container.sh reads it -# from CA_TRUST_DIR by this name (HOST_CA_BUNDLE) and generates the Java -# PKCS#12 trust store in-container from it. -readonly CA_TRUST_BUNDLE_FILENAME="ca-bundle.pem" - -# In-container mount point for the staged trust directory. Callers bind-mount -# the staging directory here (writable) and pass it as CA_TRUST_DIR so -# build-artifacts.sh / container.sh consume the staged PEM bundle and write the -# generated Java PKCS#12 trust store back into it. Kept in host.sh so the path -# is defined once on the host side rather than hardcoded by each caller. -readonly CA_TRUST_CONTAINER_DIR="/run/ca-trust" - -# Copies to , dropping any certificate whose validity period has -# already ended. A stale corporate CA bundle otherwise gets propagated -# verbatim into CURL_CA_BUNDLE, where OpenSSL (unlike macOS's SecureTransport) -# treats the file as the exclusive trust store: one expired cert anywhere in -# it is enough to break TLS verification for any download whose chain happens -# to rely on it, even though the destination server's own certificate is -# fine. Falls back to a plain copy if openssl isn't on the host, so this never -# becomes a new hard dependency. -# -# Args: -_stage_ca_bundle_without_expired_certs() { - local src="$1" dest="$2" - if ! command -v openssl &>/dev/null; then - cp "${src}" "${dest}" - return - fi - - local total=0 dropped=0 - local cert="" line - : > "${dest}" - while IFS= read -r line || [[ -n "${line}" ]]; do - cert+="${line}"$'\n' - if [[ "${line}" == "-----END CERTIFICATE-----" ]]; then - total=$((total + 1)) - if printf '%s' "${cert}" | openssl x509 -noout -checkend 0 &>/dev/null; then - printf '%s' "${cert}" >> "${dest}" - else - dropped=$((dropped + 1)) - fi - cert="" - fi - done < "${src}" - - if (( dropped > 0 )); then - echo >&2 "==> Dropped ${dropped}/${total} expired certificate(s) from host CA bundle" - fi -} - -# Stages the host CA bundle at /${CA_TRUST_BUNDLE_FILENAME} for a -# temporary Docker mount. Creates an empty file when the host has no CA bundle; -# returns nonzero only on an error. -# -# Args: -stage_host_ca_bundle() { - if (( $# != 1 )) || [[ -z "$1" ]]; then - echo >&2 "ERROR: stage_host_ca_bundle requires a trust directory" - return 2 - fi - local trust_dir="$1" - if [[ -L "${trust_dir}" || ( -e "${trust_dir}" && ! -d "${trust_dir}" ) ]]; then - echo >&2 "ERROR: stage_host_ca_bundle target is not a directory: ${trust_dir}" - return 1 - fi - if ! mkdir -p "${trust_dir}"; then - echo >&2 "ERROR: failed to create trust directory: ${trust_dir}" - return 1 - fi - trust_dir="$(cd "${trust_dir}" &>/dev/null && pwd)" || return - local dest="${trust_dir}/${CA_TRUST_BUNDLE_FILENAME}" - if [[ -L "${dest}" || ( -e "${dest}" && ! -f "${dest}" ) ]]; then - echo >&2 "ERROR: host CA bundle destination is not a regular file: ${dest}" - return 1 - fi - local source_path="" - local candidates=() - - if [[ -n "${SSL_CERT_FILE:-}" ]]; then - if [[ ! -f "${SSL_CERT_FILE}" || ! -s "${SSL_CERT_FILE}" ]]; then - echo >&2 "ERROR: SSL_CERT_FILE is not a nonempty regular file: ${SSL_CERT_FILE}" - return 1 - fi - candidates=("${SSL_CERT_FILE}") - else - candidates=( - /etc/ssl/certs/ca-certificates.crt - /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem - /etc/pki/tls/certs/ca-bundle.crt - /etc/ssl/ca-bundle.pem - /etc/pki/tls/cacert.pem - /etc/ssl/cert.pem - ) - fi - - local candidate - for candidate in "${candidates[@]}"; do - if [[ -f "${candidate}" && -s "${candidate}" ]]; then - source_path="${candidate}" - break - fi - done - - if [[ -n "${source_path}" && -e "${dest}" && "${source_path}" -ef "${dest}" ]]; then - echo >&2 "ERROR: host CA bundle source and destination must differ: ${dest}" - return 1 - fi - - local staged_bundle - if ! staged_bundle="$(mktemp "${trust_dir}/.ca-bundle.XXXXXX")"; then - echo >&2 "ERROR: failed to create temporary host CA bundle in: ${trust_dir}" - return 1 - fi - if [[ -n "${source_path}" ]]; then - echo >&2 "==> Staging host CA bundle: ${source_path} -> ${dest}" - if ! _stage_ca_bundle_without_expired_certs "${source_path}" "${staged_bundle}"; then - rm -f "${staged_bundle}" - echo >&2 "ERROR: failed to stage host CA bundle: ${source_path}" - return 1 - fi - else - echo >&2 "==> No host CA bundle found; continuing without host CA context." - fi - - # BuildKit and runtime containers consume the staged bundle read-only. - if ! chmod 0444 "${staged_bundle}"; then - rm -f "${staged_bundle}" - echo >&2 "ERROR: failed to set host CA bundle permissions: ${dest}" - return 1 - fi - if ! mv -f "${staged_bundle}" "${dest}"; then - rm -f "${staged_bundle}" - echo >&2 "ERROR: failed to replace host CA bundle: ${dest}" - return 1 - fi -} diff --git a/tools/build-packages/internal/container/build-artifacts.sh b/tools/build-packages/internal/container/build-artifacts.sh index 088da06..99debca 100755 --- a/tools/build-packages/internal/container/build-artifacts.sh +++ b/tools/build-packages/internal/container/build-artifacts.sh @@ -33,7 +33,7 @@ fi # through corporate TLS gateways. CI invokes this script directly without # CA_TRUST_DIR, so this is local-only. Sourced before MAVEN_OPTS is read below. if [[ -n "${CA_TRUST_DIR:-}" ]]; then - source "${src}/tools/build-packages/internal/ca-trust/container.sh" + source "${src}/tools/yscope-dev-utils/exports/docker/ca-trust/container.sh" fi # Destination paths used by .deb and .rpm. Environment overrides support a diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 0c214c4..060c45c 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 0c214c44acddff330a204201428ea145e641891d +Subproject commit 060c45c24f2bcd92e4af69b9360f253edc11ba80