Skip to content

fix: propagate dependency generation failures #736

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bdice wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

fix: propagate dependency generation failures #736

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion features/src/rapids-build-utils/devcontainer-feature.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "NVIDIA RAPIDS devcontainer build utilities",
"id": "rapids-build-utils",
"version": "26.8.7",
"version": "26.8.8",
"description": "A feature to install the RAPIDS devcontainer build utilities",
"containerEnv": {
"BASH_ENV": "/etc/bash.bash_env"
Expand Down
111 changes: 100 additions & 11 deletions features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,45 @@
. rapids-generate-docstring;

_generate_env_yaml() {
if rapids-dependency-file-generator "$@" 2>/dev/null \
| grep -v '^#' 2>/dev/null \
| tee "$file" 1>/dev/null; then
echo "$file"
local file="$1";
local status;
shift;

if rapids-dependency-file-generator "$@" > "${file}.generated"; then
if sed '/^#/d' "${file}.generated" > "$file"; then
return 0;
else
status=$?;
echo "Failed to filter generated conda environment for ${file##*/}" 1>&2;
return "${status}";
fi
else
status=$?;
echo "Failed to generate conda environment for ${file##*/}" 1>&2;
return "${status}";
fi
}

_list_contains() {
local requested_value="$1";
shift;

local value;
for value in "$@"; do
if test "${requested_value}" == "${value}"; then
return 0;
fi
done

return 1;
}

_generate_env_yamls() {
local i;
local j;
local generation_failed=0;
local -a generated_files=();
local -a generation_pids=();

for ((i=0; i < ${repos_length:-0}; i+=1)); do

Expand All @@ -48,6 +77,20 @@ _generate_env_yamls() {
&& test -n "${path:+x}" \
&& test -f ~/"${path}/dependencies.yaml"; then

local config=~/"${path}/dependencies.yaml";
local available_file_keys_output;
local -a available_file_keys=();

if available_file_keys_output="$(yq -r '.files | keys | .[]' "${config}")"; then
if test -n "${available_file_keys_output:+x}"; then
readarray -t available_file_keys <<< "${available_file_keys_output}";
fi
else
echo "Failed to read dependency file keys from ${config}" 1>&2;
generation_failed=1;
continue;
fi

echo "Generating ${name}'s repo conda env yml" 1>&2;

local dependency_keys=("${key[@]}");
Expand All @@ -69,12 +112,20 @@ _generate_env_yamls() {
local keyi;

for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then
continue;
fi
local file="${tmpdir}/${name}.${keys[$keyi]}.env.yaml";
_generate_env_yaml \
if _list_contains "${file}" "${generated_files[@]}"; then
continue;
fi
generated_files+=("${file}");
_generate_env_yaml "${file}" \
--file-key "${keys[$keyi]}" \
--output conda \
--config ~/"${path}/dependencies.yaml" \
--config "${config}" \
--matrix "${matrix_selectors}" &
generation_pids+=("$!");
done

local cpp_length="${repo}_cpp_length";
Expand All @@ -89,12 +140,20 @@ _generate_env_yamls() {
local keyi;

for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then
continue;
fi
local file="${tmpdir}/${name}.${cpp_name}.${keys[$keyi]}.env.yaml";
_generate_env_yaml \
if _list_contains "${file}" "${generated_files[@]}"; then
continue;
fi
generated_files+=("${file}");
_generate_env_yaml "${file}" \
--file-key "${keys[$keyi]}" \
--output conda \
--config ~/"${path}/dependencies.yaml" \
--config "${config}" \
--matrix "${matrix_selectors}" &
generation_pids+=("$!");
done
done

Expand All @@ -110,16 +169,43 @@ _generate_env_yamls() {
local keyi;

for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then
continue;
fi
local file="${tmpdir}/${name}.${py_name}.${keys[$keyi]}.env.yaml";
_generate_env_yaml \
if _list_contains "${file}" "${generated_files[@]}"; then
continue;
fi
generated_files+=("${file}");
_generate_env_yaml "${file}" \
--file-key "${keys[$keyi]}" \
--output conda \
--config ~/"${path}/dependencies.yaml" \
--config "${config}" \
--matrix "${matrix_selectors}" &
generation_pids+=("$!");
done
done
fi
done

local pid;
for pid in "${generation_pids[@]}"; do
if ! wait "${pid}"; then
generation_failed=1;
fi
done

if test "${generation_failed}" -ne 0; then
echo "Failed to generate one or more conda environment files" 1>&2;
return 1;
fi

local file;
for file in "${generated_files[@]}"; do
if test -s "${file}"; then
conda_env_yamls+=("${file}");
fi
done
}

_make_conda_dependencies() {
Expand Down Expand Up @@ -169,7 +255,10 @@ _make_conda_dependencies() {
# shellcheck disable=SC2064
trap "rm -rf '${tmpdir}'" EXIT;

readarray -t conda_env_yamls < <(_generate_env_yamls);
local -a conda_env_yamls=();
if ! _generate_env_yamls; then
return 1;
fi

if test ${#conda_env_yamls[@]} -gt 0; then

Expand Down
114 changes: 100 additions & 14 deletions features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,45 @@
. rapids-generate-docstring;

_generate_requirements_txt() {
if rapids-dependency-file-generator "$@" 2>/dev/null \
| grep -v '^#' 2>/dev/null \
| tee "$file" 1>/dev/null; then
echo "$file"
local file="$1";
local status;
shift;

if rapids-dependency-file-generator "$@" > "${file}.generated"; then
if sed '/^#/d' "${file}.generated" > "$file"; then
return 0;
else
status=$?;
echo "Failed to filter generated requirements for ${file##*/}" 1>&2;
return "${status}";
fi
else
status=$?;
echo "Failed to generate requirements for ${file##*/}" 1>&2;
return "${status}";
fi
}

_list_contains() {
local requested_value="$1";
shift;

local value;
for value in "$@"; do
if test "${requested_value}" == "${value}"; then
return 0;
fi
done

return 1;
}

_generate_requirements_txts() {
local i;
local j;
local generation_failed=0;
local -a generated_files=();
local -a generation_pids=();

for ((i=0; i < ${repos_length:-0}; i+=1)); do

Expand All @@ -50,6 +79,20 @@ _generate_requirements_txts() {
&& test -n "${path:+x}" \
&& test -f ~/"${path}/dependencies.yaml"; then

local config=~/"${path}/dependencies.yaml";
local available_file_keys_output;
local -a available_file_keys=();

if available_file_keys_output="$(yq -r '.files | keys | .[]' "${config}")"; then
if test -n "${available_file_keys_output:+x}"; then
readarray -t available_file_keys <<< "${available_file_keys_output}";
fi
else
echo "Failed to read dependency file keys from ${config}" 1>&2;
generation_failed=1;
continue;
fi

echo "Generating ${name}'s repo requirements.txt" 1>&2;

local dependency_keys=("${key[@]}");
Expand All @@ -71,13 +114,20 @@ _generate_requirements_txts() {
local keyi;

for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then
continue;
fi
local file="${tmpdir}/${name}.${keys[$keyi]}.requirements.txt";
pip_reqs_txts+=("${file}");
_generate_requirements_txt \
if _list_contains "${file}" "${generated_files[@]}"; then
continue;
fi
generated_files+=("${file}");
_generate_requirements_txt "${file}" \
--file-key "${keys[$keyi]}" \
--output requirements \
--config ~/"${path}/dependencies.yaml" \
--config "${config}" \
--matrix "${matrix_selectors}" &
generation_pids+=("$!");
done

local cpp_length="${repo}_cpp_length";
Expand All @@ -92,13 +142,20 @@ _generate_requirements_txts() {
local keyi;

for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then
continue;
fi
local file="${tmpdir}/${name}.${cpp_name}.${keys[$keyi]}.requirements.txt";
pip_reqs_txts+=("${file}");
_generate_requirements_txt \
if _list_contains "${file}" "${generated_files[@]}"; then
continue;
fi
generated_files+=("${file}");
_generate_requirements_txt "${file}" \
--file-key "${keys[$keyi]}" \
--output requirements \
--config ~/"${path}/dependencies.yaml" \
--config "${config}" \
--matrix "${matrix_selectors}" &
generation_pids+=("$!");
done
done

Expand All @@ -114,17 +171,43 @@ _generate_requirements_txts() {
local keyi;

for ((keyi=0; keyi < ${#keys[@]}; keyi+=1)); do
if ! _list_contains "${keys[$keyi]}" "${available_file_keys[@]}"; then
continue;
fi
local file="${tmpdir}/${name}.${py_name}.${keys[$keyi]}.requirements.txt";
pip_reqs_txts+=("${file}");
_generate_requirements_txt \
if _list_contains "${file}" "${generated_files[@]}"; then
continue;
fi
generated_files+=("${file}");
_generate_requirements_txt "${file}" \
--file-key "${keys[$keyi]}" \
--output requirements \
--config ~/"${path}/dependencies.yaml" \
--config "${config}" \
--matrix "${matrix_selectors}" &
generation_pids+=("$!");
done
done
fi
done

local pid;
for pid in "${generation_pids[@]}"; do
if ! wait "${pid}"; then
generation_failed=1;
fi
done

if test "${generation_failed}" -ne 0; then
echo "Failed to generate one or more requirements files" 1>&2;
return 1;
fi

local file;
for file in "${generated_files[@]}"; do
if test -s "${file}"; then
pip_reqs_txts+=("${file}");
fi
done
}

_make_pip_dependencies() {
Expand Down Expand Up @@ -188,7 +271,10 @@ _make_pip_dependencies() {

eval "$(rapids-list-repos "${OPTS[@]}")";

readarray -t pip_reqs_txts < <(_generate_requirements_txts);
local -a pip_reqs_txts=();
if ! _generate_requirements_txts; then
return 1;
fi

if test ${#requirement[@]} -gt 0 || test ${#pip_reqs_txts[@]} -gt 0; then

Expand Down
Loading
Loading