diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index aed128d8..c00199b8 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "26.8.7", + "version": "26.8.8", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index e9b2ecb8..139cdb61 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -25,16 +25,45 @@ . rapids-generate-docstring; _generate_env_yaml() { - if rapids-dependency-file-generator "$@" 2>/dev/null \ - | grep -v '^#' 2>/dev/null \ - | tee "$file" 1>/dev/null; then - echo "$file" + local file="$1"; + local status; + shift; + + if rapids-dependency-file-generator "$@" > "${file}.generated"; then + if sed '/^#/d' "${file}.generated" > "$file"; then + return 0; + else + status=$?; + echo "Failed to filter generated conda environment for ${file##*/}" 1>&2; + return "${status}"; + fi + else + status=$?; + echo "Failed to generate conda environment for ${file##*/}" 1>&2; + return "${status}"; fi } +_list_contains() { + local requested_value="$1"; + shift; + + local value; + for value in "$@"; do + if test "${requested_value}" == "${value}"; then + return 0; + fi + done + + return 1; +} + _generate_env_yamls() { local i; local j; + local generation_failed=0; + local -a generated_files=(); + local -a generation_pids=(); for ((i=0; i < ${repos_length:-0}; i+=1)); do @@ -48,6 +77,20 @@ _generate_env_yamls() { && test -n "${path:+x}" \ && test -f ~/"${path}/dependencies.yaml"; then + local config=~/"${path}/dependencies.yaml"; + local available_file_keys_output; + local -a available_file_keys=(); + + if available_file_keys_output="$(yq -r '.files | keys | .[]' "${config}")"; then + if test -n "${available_file_keys_output:+x}"; then + readarray -t available_file_keys <<< "${available_file_keys_output}"; + fi + else + echo "Failed to read dependency file keys from ${config}" 1>&2; + generation_failed=1; + continue; + fi + echo "Generating ${name}'s repo conda env yml" 1>&2; local dependency_keys=("${key[@]}"); @@ -69,12 +112,20 @@ _generate_env_yamls() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then + continue; + fi local file="${tmpdir}/${name}.${keys[$keyi]}.env.yaml"; - _generate_env_yaml \ + if _list_contains "${file}" "${generated_files[@]}"; then + continue; + fi + generated_files+=("${file}"); + _generate_env_yaml "${file}" \ --file-key "${keys[$keyi]}" \ --output conda \ - --config ~/"${path}/dependencies.yaml" \ + --config "${config}" \ --matrix "${matrix_selectors}" & + generation_pids+=("$!"); done local cpp_length="${repo}_cpp_length"; @@ -89,12 +140,20 @@ _generate_env_yamls() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then + continue; + fi local file="${tmpdir}/${name}.${cpp_name}.${keys[$keyi]}.env.yaml"; - _generate_env_yaml \ + if _list_contains "${file}" "${generated_files[@]}"; then + continue; + fi + generated_files+=("${file}"); + _generate_env_yaml "${file}" \ --file-key "${keys[$keyi]}" \ --output conda \ - --config ~/"${path}/dependencies.yaml" \ + --config "${config}" \ --matrix "${matrix_selectors}" & + generation_pids+=("$!"); done done @@ -110,16 +169,43 @@ _generate_env_yamls() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then + continue; + fi local file="${tmpdir}/${name}.${py_name}.${keys[$keyi]}.env.yaml"; - _generate_env_yaml \ + if _list_contains "${file}" "${generated_files[@]}"; then + continue; + fi + generated_files+=("${file}"); + _generate_env_yaml "${file}" \ --file-key "${keys[$keyi]}" \ --output conda \ - --config ~/"${path}/dependencies.yaml" \ + --config "${config}" \ --matrix "${matrix_selectors}" & + generation_pids+=("$!"); done done fi done + + local pid; + for pid in "${generation_pids[@]}"; do + if ! wait "${pid}"; then + generation_failed=1; + fi + done + + if test "${generation_failed}" -ne 0; then + echo "Failed to generate one or more conda environment files" 1>&2; + return 1; + fi + + local file; + for file in "${generated_files[@]}"; do + if test -s "${file}"; then + conda_env_yamls+=("${file}"); + fi + done } _make_conda_dependencies() { @@ -169,7 +255,10 @@ _make_conda_dependencies() { # shellcheck disable=SC2064 trap "rm -rf '${tmpdir}'" EXIT; - readarray -t conda_env_yamls < <(_generate_env_yamls); + local -a conda_env_yamls=(); + if ! _generate_env_yamls; then + return 1; + fi if test ${#conda_env_yamls[@]} -gt 0; then diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 41886b0e..457b9085 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -27,16 +27,45 @@ . rapids-generate-docstring; _generate_requirements_txt() { - if rapids-dependency-file-generator "$@" 2>/dev/null \ - | grep -v '^#' 2>/dev/null \ - | tee "$file" 1>/dev/null; then - echo "$file" + local file="$1"; + local status; + shift; + + if rapids-dependency-file-generator "$@" > "${file}.generated"; then + if sed '/^#/d' "${file}.generated" > "$file"; then + return 0; + else + status=$?; + echo "Failed to filter generated requirements for ${file##*/}" 1>&2; + return "${status}"; + fi + else + status=$?; + echo "Failed to generate requirements for ${file##*/}" 1>&2; + return "${status}"; fi } +_list_contains() { + local requested_value="$1"; + shift; + + local value; + for value in "$@"; do + if test "${requested_value}" == "${value}"; then + return 0; + fi + done + + return 1; +} + _generate_requirements_txts() { local i; local j; + local generation_failed=0; + local -a generated_files=(); + local -a generation_pids=(); for ((i=0; i < ${repos_length:-0}; i+=1)); do @@ -50,6 +79,20 @@ _generate_requirements_txts() { && test -n "${path:+x}" \ && test -f ~/"${path}/dependencies.yaml"; then + local config=~/"${path}/dependencies.yaml"; + local available_file_keys_output; + local -a available_file_keys=(); + + if available_file_keys_output="$(yq -r '.files | keys | .[]' "${config}")"; then + if test -n "${available_file_keys_output:+x}"; then + readarray -t available_file_keys <<< "${available_file_keys_output}"; + fi + else + echo "Failed to read dependency file keys from ${config}" 1>&2; + generation_failed=1; + continue; + fi + echo "Generating ${name}'s repo requirements.txt" 1>&2; local dependency_keys=("${key[@]}"); @@ -71,13 +114,20 @@ _generate_requirements_txts() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then + continue; + fi local file="${tmpdir}/${name}.${keys[$keyi]}.requirements.txt"; - pip_reqs_txts+=("${file}"); - _generate_requirements_txt \ + if _list_contains "${file}" "${generated_files[@]}"; then + continue; + fi + generated_files+=("${file}"); + _generate_requirements_txt "${file}" \ --file-key "${keys[$keyi]}" \ --output requirements \ - --config ~/"${path}/dependencies.yaml" \ + --config "${config}" \ --matrix "${matrix_selectors}" & + generation_pids+=("$!"); done local cpp_length="${repo}_cpp_length"; @@ -92,13 +142,20 @@ _generate_requirements_txts() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then + continue; + fi local file="${tmpdir}/${name}.${cpp_name}.${keys[$keyi]}.requirements.txt"; - pip_reqs_txts+=("${file}"); - _generate_requirements_txt \ + if _list_contains "${file}" "${generated_files[@]}"; then + continue; + fi + generated_files+=("${file}"); + _generate_requirements_txt "${file}" \ --file-key "${keys[$keyi]}" \ --output requirements \ - --config ~/"${path}/dependencies.yaml" \ + --config "${config}" \ --matrix "${matrix_selectors}" & + generation_pids+=("$!"); done done @@ -114,17 +171,43 @@ _generate_requirements_txts() { local keyi; for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do + if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then + continue; + fi local file="${tmpdir}/${name}.${py_name}.${keys[$keyi]}.requirements.txt"; - pip_reqs_txts+=("${file}"); - _generate_requirements_txt \ + if _list_contains "${file}" "${generated_files[@]}"; then + continue; + fi + generated_files+=("${file}"); + _generate_requirements_txt "${file}" \ --file-key "${keys[$keyi]}" \ --output requirements \ - --config ~/"${path}/dependencies.yaml" \ + --config "${config}" \ --matrix "${matrix_selectors}" & + generation_pids+=("$!"); done done fi done + + local pid; + for pid in "${generation_pids[@]}"; do + if ! wait "${pid}"; then + generation_failed=1; + fi + done + + if test "${generation_failed}" -ne 0; then + echo "Failed to generate one or more requirements files" 1>&2; + return 1; + fi + + local file; + for file in "${generated_files[@]}"; do + if test -s "${file}"; then + pip_reqs_txts+=("${file}"); + fi + done } _make_pip_dependencies() { @@ -188,7 +271,10 @@ _make_pip_dependencies() { eval "$(rapids-list-repos "${OPTS[@]}")"; - readarray -t pip_reqs_txts < <(_generate_requirements_txts); + local -a pip_reqs_txts=(); + if ! _generate_requirements_txts; then + return 1; + fi if test ${#requirement[@]} -gt 0 || test ${#pip_reqs_txts[@]} -gt 0; then diff --git a/features/test/rapids-build-utils/test.sh b/features/test/rapids-build-utils/test.sh index d2593026..c6eaaab5 100644 --- a/features/test/rapids-build-utils/test.sh +++ b/features/test/rapids-build-utils/test.sh @@ -2,11 +2,214 @@ set -e -# Optional: Import test library source dev-container-features-test-lib -# Definition specific tests -check "TODO" echo "success" +test_root="$(mktemp -d)" +trap 'rm -rf "${test_root}"' EXIT + +mkdir -p "${test_root}/bin" "${test_root}/home/test-repo" + +cat > "${test_root}/bin/devcontainer-utils-parse-args" <<'EOF' +#!/bin/bash + +_print_usage() { + : +} + +_parse_args_for_file() { + shift + + local -a parsed_keys=() + while test "$#" -gt 0; do + case "$1" in + --skip|--take) + shift 2 + ;; + -k|--key) + parsed_keys+=("$2") + shift 2 + ;; + -m|--manifest) + shift 2 + ;; + *) + shift + ;; + esac + done + + cat <<'DECLARATIONS' +declare -a OPTS=() +declare -a exclude=() +declare -a include=() +declare -a key=() +declare -a matrix_entry=() +declare -a requirement=() +DECLARATIONS + + local parsed_key + for parsed_key in "${parsed_keys[@]}"; do + printf 'key+=(%q)\n' "${parsed_key}" + done +} +EOF +chmod +x "${test_root}/bin/devcontainer-utils-parse-args" + +cat > "${test_root}/bin/devcontainer-utils-debug-output" <<'EOF' +#!/bin/bash +: +EOF +chmod +x "${test_root}/bin/devcontainer-utils-debug-output" + +cat > "${test_root}/bin/rapids-list-repos" <<'EOF' +#!/bin/bash +cat <<'REPOS' +declare repos_length=1 +declare repos_0_name=dfg-atomic +declare repos_0_path=test-repo +REPOS +EOF +chmod +x "${test_root}/bin/rapids-list-repos" + +cat > "${test_root}/bin/rapids-python-pkg-names" <<'EOF' +#!/bin/bash +: +EOF +chmod +x "${test_root}/bin/rapids-python-pkg-names" + +cat > "${test_root}/bin/rapids-python-conda-pkg-names" <<'EOF' +#!/bin/bash +: +EOF +chmod +x "${test_root}/bin/rapids-python-conda-pkg-names" + +cat > "${test_root}/bin/rapids-dependency-file-generator" <<'EOF' +#!/bin/bash + +set -euo pipefail + +file_key= +output= + +while test "$#" -gt 0; do + case "$1" in + --file-key) + file_key="$2" + shift 2 + ;; + --output) + output="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +printf '%s\n' "${file_key}" >> "${DFG_CALL_LOG}" + +if test "${file_key}" == hard_error; then + echo partial-output-must-not-escape + echo 'mock hard generation failure' 1>&2 + exit 42 +fi + +case "${output}" in + requirements) + cat <<'REQUIREMENTS' +# generated +dfg-atomic-success +REQUIREMENTS + ;; + conda) + cat <<'CONDA' +# generated +name: dfg-atomic +channels: + - conda-forge +dependencies: + - dfg-atomic-success +CONDA + ;; + *) + echo "unexpected output type: ${output}" 1>&2 + exit 2 + ;; +esac +EOF +chmod +x "${test_root}/bin/rapids-dependency-file-generator" + +cat > "${test_root}/manifest.yaml" <<'EOF' +repos: + - name: dfg-atomic + path: test-repo +EOF + +cat > "${test_root}/home/test-repo/dependencies.yaml" <<'EOF' +files: + success: {} + hard_error: {} +EOF + +dependencies_skip_missing_key() { + local command="$1" + local label="$2" + local stdout="${test_root}/${label}-missing.stdout" + local stderr="${test_root}/${label}-missing.stderr" + local calls="${test_root}/${label}-missing.calls" + + : > "${calls}" + if ! env \ + PATH="${test_root}/bin:${PATH}" \ + HOME="${test_root}/home" \ + DFG_CALL_LOG="${calls}" \ + "${command}" \ + --manifest "${test_root}/manifest.yaml" \ + --key success \ + --key missing \ + --key success \ + > "${stdout}" 2> "${stderr}"; then + return 1 + fi + + test -s "${stdout}" \ + && grep -Fq dfg-atomic-success "${stdout}" \ + && test "$(wc -l < "${calls}")" -eq 1 \ + && grep -Fxq success "${calls}" \ + && ! grep -Fxq missing "${calls}" +} + +dependencies_fail_atomically() { + local command="$1" + local label="$2" + local stdout="${test_root}/${label}-failure.stdout" + local stderr="${test_root}/${label}-failure.stderr" + local calls="${test_root}/${label}-failure.calls" + + : > "${calls}" + if env \ + PATH="${test_root}/bin:${PATH}" \ + HOME="${test_root}/home" \ + DFG_CALL_LOG="${calls}" \ + "${command}" \ + --manifest "${test_root}/manifest.yaml" \ + --key success \ + --key hard_error \ + > "${stdout}" 2> "${stderr}"; then + return 1 + fi + + test ! -s "${stdout}" \ + && grep -Fq 'mock hard generation failure' "${stderr}" \ + && test "$(wc -l < "${calls}")" -eq 2 \ + && grep -Fxq success "${calls}" \ + && grep -Fxq hard_error "${calls}" +} + +check "pip dependencies skip missing keys" dependencies_skip_missing_key rapids-make-pip-dependencies pip +check "pip dependency failures are atomic" dependencies_fail_atomically rapids-make-pip-dependencies pip +check "conda dependencies skip missing keys" dependencies_skip_missing_key rapids-make-conda-dependencies conda +check "conda dependency failures are atomic" dependencies_fail_atomically rapids-make-conda-dependencies conda -# Report result reportResults