Skip to content

Commit a4a6b83

Browse files
jameslambjameslamb
authored
combine pip install calls in wheel-testing scripts (#4701)
## Summary Follow-up to #4690. Proposes consolidating stuff like this in CI scripts: ```shell pip install A pip install B pip install C ``` Into this: ```shell pip install A B C ``` ## Benefits of these changes Reduces the risk of creating a broken environment with incompatible packages. Unlike `conda`, `pip` does not evaluate the requirements of all installed packages when you run `pip` install. Installing `torch` and `cugraph-dgl` at the same time, for example, gives us a chance to find out about packaging issues like *"`cugraph-dgl` and `torch` have conflicting requirements on `{other_package}`"* at CI time. Similar change from `cudf`: rapidsai/cudf#16575 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Alex Barghi (https://github.com/alexbarghi-nv) URL: #4701
1 parent 6f4689a commit a4a6b83

7 files changed

Lines changed: 49 additions & 53 deletions

‎ci/test_wheel.sh‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
set -eoxu pipefail
55

66
package_name=$1
7-
package_dir=$2
87

98
python_package_name=$(echo ${package_name}|sed 's/-/_/g')
109

‎ci/test_wheel_cugraph-dgl.sh‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,16 @@
44
set -eoxu pipefail
55

66
package_name="cugraph-dgl"
7-
package_dir="python/cugraph-dgl"
8-
9-
python_package_name=$(echo ${package_name}|sed 's/-/_/g')
107

118
mkdir -p ./dist
129
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
1310

14-
# Download wheels built during this job.
11+
# Download the pylibcugraph, cugraph, and cugraph-dgl built in the previous step
1512
RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
1613
RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
17-
python -m pip install ./local-deps/*.whl
18-
19-
# use 'ls' to expand wildcard before adding `[extra]` requires for pip
2014
RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
21-
# pip creates wheels using python package names
22-
python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
23-
2415

16+
# determine pytorch and DGL sources
2517
PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')"
2618
PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2}
2719
if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then
@@ -32,8 +24,15 @@ fi
3224
PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}"
3325
DGL_URL="https://data.dgl.ai/wheels/torch-2.3/cu${PYTORCH_CUDA_VER}/repo.html"
3426

35-
rapids-logger "Installing PyTorch and DGL"
36-
rapids-retry python -m pip install torch==2.3.0 --index-url ${PYTORCH_URL}
37-
rapids-retry python -m pip install dgl==2.4.0 --find-links ${DGL_URL}
27+
# echo to expand wildcard before adding `[extra]` requires for pip
28+
python -m pip install \
29+
-v \
30+
--extra-index-url "${PYTORCH_URL}" \
31+
--find-links "${DGL_URL}" \
32+
"$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
33+
"$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
34+
"$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
35+
'dgl==2.4.0' \
36+
'torch>=2.3.0,<2.4'
3837

3938
python -m pytest python/cugraph-dgl/tests

‎ci/test_wheel_cugraph-equivariant.sh‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,14 @@
44
set -eoxu pipefail
55

66
package_name="cugraph-equivariant"
7-
package_dir="python/cugraph-equivariant"
8-
9-
python_package_name=$(echo ${package_name}|sed 's/-/_/g')
107

118
mkdir -p ./dist
129
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
1310

14-
# use 'ls' to expand wildcard before adding `[extra]` requires for pip
11+
# Download the cugraph-equivariant built in the previous step
1512
RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
16-
# pip creates wheels using python package names
17-
python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
18-
1913

14+
# determine pytorch source
2015
PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')"
2116
PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2}
2217
if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then
@@ -26,8 +21,12 @@ else
2621
fi
2722
PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}"
2823

29-
rapids-logger "Installing PyTorch and e3nn"
30-
rapids-retry python -m pip install torch --index-url ${PYTORCH_URL}
31-
rapids-retry python -m pip install e3nn
24+
# echo to expand wildcard before adding `[extra]` requires for pip
25+
python -m pip install \
26+
-v \
27+
--extra-index-url "${PYTORCH_URL}" \
28+
"$(echo ./dist/cugraph_equivariant_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
29+
'e3nn' \
30+
'torch>=2.3.0,<2.4'
3231

3332
python -m pytest python/cugraph-equivariant/cugraph_equivariant/tests

‎ci/test_wheel_cugraph-pyg.sh‎

Lines changed: 23 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,45 +4,44 @@
44
set -eoxu pipefail
55

66
package_name="cugraph-pyg"
7-
package_dir="python/cugraph-pyg"
8-
9-
python_package_name=$(echo ${package_name}|sed 's/-/_/g')
107

118
mkdir -p ./dist
129
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
1310

14-
# Download wheels built during this job.
11+
# Download the pylibcugraph, cugraph, and cugraph-pyg built in the previous step
1512
RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
1613
RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
17-
python -m pip install ./local-deps/*.whl
18-
19-
# use 'ls' to expand wildcard before adding `[extra]` requires for pip
2014
RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist
21-
# pip creates wheels using python package names
22-
python -m pip install $(ls ./dist/${python_package_name}*.whl)[test]
23-
24-
# RAPIDS_DATASET_ROOT_DIR is used by test scripts
25-
export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
26-
27-
# Used to skip certain examples in CI due to memory limitations
28-
export CI_RUN=1
2915

16+
# determine pytorch and pyg sources
3017
if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then
3118
PYTORCH_URL="https://download.pytorch.org/whl/cu118"
3219
PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html"
3320
else
3421
PYTORCH_URL="https://download.pytorch.org/whl/cu121"
3522
PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html"
3623
fi
37-
rapids-logger "Installing PyTorch and PyG dependencies"
38-
rapids-retry python -m pip install torch==2.3.0 --index-url ${PYTORCH_URL}
39-
rapids-retry python -m pip install "torch-geometric>=2.5,<2.6"
40-
rapids-retry python -m pip install \
41-
ogb \
42-
pyg_lib \
43-
torch_scatter \
44-
torch_sparse \
45-
-f ${PYG_URL}
24+
25+
# echo to expand wildcard before adding `[extra]` requires for pip
26+
python -m pip install \
27+
-v \
28+
--extra-index-url "${PYTORCH_URL}" \
29+
--find-links "${PYG_URL}" \
30+
"$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
31+
"$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
32+
"$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \
33+
'ogb' \
34+
'pyg_lib' \
35+
'torch>=2.3.0,<2.4' \
36+
'torch-geometric>=2.5,<2.6' \
37+
'torch_scatter' \
38+
'torch_sparse'
39+
40+
# RAPIDS_DATASET_ROOT_DIR is used by test scripts
41+
export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
42+
43+
# Used to skip certain examples in CI due to memory limitations
44+
export CI_RUN=1
4645

4746
rapids-logger "pytest cugraph-pyg (single GPU)"
4847
pushd python/cugraph-pyg/cugraph_pyg

‎ci/test_wheel_cugraph.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# Copyright (c) 2023, NVIDIA CORPORATION.
2+
# Copyright (c) 2023-2024, NVIDIA CORPORATION.
33

44
set -eoxu pipefail
55

@@ -8,4 +8,4 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
88
RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep
99
python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl
1010

11-
./ci/test_wheel.sh cugraph python/cugraph
11+
./ci/test_wheel.sh cugraph

‎ci/test_wheel_nx-cugraph.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
88
RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps
99
python -m pip install ./local-deps/*.whl
1010

11-
./ci/test_wheel.sh nx-cugraph python/nx-cugraph
11+
./ci/test_wheel.sh nx-cugraph

‎ci/test_wheel_pylibcugraph.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
2-
# Copyright (c) 2023, NVIDIA CORPORATION.
2+
# Copyright (c) 2023-2024, NVIDIA CORPORATION.
33

44
set -eoxu pipefail
55

6-
./ci/test_wheel.sh pylibcugraph python/pylibcugraph
6+
./ci/test_wheel.sh pylibcugraph

0 commit comments

Comments
 (0)