Skip to content

Commit c96bd6b

Browse files
mhaseeb123mhaseeb123
authored
Merge branch 'main' into fea/py-bind-cudf-apply-deletion-mask
2 parents 409858b + 563021f commit c96bd6b

423 files changed

Lines changed: 3809 additions & 5208 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.devcontainer/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@ ARG PYTHON_PACKAGE_MANAGER=conda
55

66
FROM ${BASE} as pip-base
77

8+
# libnuma-dev is required for pip devcontainers for cucascade from rapidsmpf
89
RUN apt update -y \
910
&& DEBIAN_FRONTEND=noninteractive apt install -y \
1011
librdkafka-dev \
12+
libnuma-dev \
1113
&& rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*;
1214

1315
ENV DEFAULT_VIRTUAL_ENV=rapids

‎.devcontainer/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ This container is a turnkey development environment for building and testing the
2020
By default, the following directories are bind-mounted into the devcontainer:
2121

2222
* `${repo}:/home/coder/cudf`
23+
* `${repo}/../rapidsmpf:/home/coder/rapidsmpf`
2324
* `${repo}/../.aws:/home/coder/.aws`
2425
* `${repo}/../.local:/home/coder/.local`
2526
* `${repo}/../.cache:/home/coder/.cache`
@@ -28,6 +29,13 @@ By default, the following directories are bind-mounted into the devcontainer:
2829

2930
This ensures caches, configurations, dependencies, and your commits are persisted on the host across container runs.
3031

32+
The [rapidsmpf](https://github.com/rapidsai/rapidsmpf) repository is a required dependency of `cudf_polars` (that also requires `libcudf`) and must be cloned as a sibling directory to the cudf repo before launching the devcontainer:
33+
34+
```
35+
# from the parent directory of your cudf clone
36+
git clone https://github.com/rapidsai/rapidsmpf.git
37+
```
38+
3139
## Launch a Dev Container
3240

3341
To launch a devcontainer from VSCode, open the cuDF repo and select the "Reopen in Container" button in the bottom right:<br/><img src="https://user-images.githubusercontent.com/178183/221771999-97ab29d5-e718-4e5f-b32f-2cdd51bba25c.png"/>

‎.devcontainer/cuda12.9-conda/devcontainer.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
"initializeCommand": [
5050
"/bin/bash",
5151
"-c",
52-
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.9-envs}"
52+
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.9-envs} ${localWorkspaceFolder}/../rapidsmpf"
5353
],
5454
"postAttachCommand": [
5555
"/bin/bash",
@@ -63,7 +63,8 @@
6363
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
6464
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
6565
"source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
66-
"source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.9-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
66+
"source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.9-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
67+
"source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent"
6768
],
6869
"customizations": {
6970
"vscode": {

‎.devcontainer/cuda12.9-pip/devcontainer.json‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"args": {
66
"CUDA": "12.9",
77
"PYTHON_PACKAGE_MANAGER": "pip",
8-
"BASE": "rapidsai/devcontainers:26.06-cpp-cuda12.9"
8+
"BASE": "rapidsai/devcontainers:26.06-cpp-cuda12.9-ucx1.19.0-openmpi5.0.10"
99
}
1010
},
1111
"runArgs": [
@@ -27,7 +27,7 @@
2727
"initializeCommand": [
2828
"/bin/bash",
2929
"-c",
30-
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs}"
30+
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs} ${localWorkspaceFolder}/../rapidsmpf"
3131
],
3232
"postAttachCommand": [
3333
"/bin/bash",
@@ -40,7 +40,8 @@
4040
"source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
4141
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
4242
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
43-
"source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent"
43+
"source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent",
44+
"source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent"
4445
],
4546
"customizations": {
4647
"vscode": {

‎.devcontainer/cuda13.1-conda/devcontainer.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
"initializeCommand": [
5050
"/bin/bash",
5151
"-c",
52-
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda13.1-envs}"
52+
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda13.1-envs} ${localWorkspaceFolder}/../rapidsmpf"
5353
],
5454
"postAttachCommand": [
5555
"/bin/bash",
@@ -63,7 +63,8 @@
6363
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
6464
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
6565
"source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
66-
"source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda13.1-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
66+
"source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda13.1-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
67+
"source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent"
6768
],
6869
"customizations": {
6970
"vscode": {

‎.devcontainer/cuda13.1-pip/devcontainer.json‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"args": {
66
"CUDA": "13.1",
77
"PYTHON_PACKAGE_MANAGER": "pip",
8-
"BASE": "rapidsai/devcontainers:26.06-cpp-cuda13.1"
8+
"BASE": "rapidsai/devcontainers:26.06-cpp-cuda13.1-ucx1.19.0-openmpi5.0.10"
99
}
1010
},
1111
"runArgs": [
@@ -27,7 +27,7 @@
2727
"initializeCommand": [
2828
"/bin/bash",
2929
"-c",
30-
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda13.1-venvs}"
30+
"mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda13.1-venvs} ${localWorkspaceFolder}/../rapidsmpf"
3131
],
3232
"postAttachCommand": [
3333
"/bin/bash",
@@ -40,7 +40,8 @@
4040
"source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
4141
"source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
4242
"source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
43-
"source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda13.1-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent"
43+
"source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda13.1-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent",
44+
"source=${localWorkspaceFolder}/../rapidsmpf,target=/home/coder/rapidsmpf,type=bind,consistency=consistent"
4445
],
4546
"customizations": {
4647
"vscode": {

‎.github/workflows/pr.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,9 +550,16 @@ jobs:
550550
SCCACHE_DIST_MAX_RETRIES=inf
551551
SCCACHE_SERVER_LOG=sccache=debug
552552
SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false
553+
# clone-rapidsmpf to not use the rapidsmpf wheels from cudf_polars dependency
554+
# librapidsmpf-cu13 wheels brings in a hardcoded libnuma-dev cmake target: https://github.com/NVIDIA/cuCascade/issues/118
555+
# -DBUILD_TESTS=OFF to match rapidsmpf https://github.com/rapidsai/rapidsmpf/blob/main/.github/workflows/pr.yaml#L351 (leads to compilation errors)
556+
# -DCUDF_BUILD_TESTUTIL=OFF to avoid IMPORTED_GLOBAL promotion errors when cuCascade's find_package(cudf) loads cudf-config.cmake from a CPM subdirectory
553557
build_command: |
554558
sccache --zero-stats;
555-
build-all -j0 -DBUILD_BENCHMARKS=ON --verbose 2>&1 | tee telemetry-artifacts/build.log;
559+
clone-rapidsmpf -j$(nproc) -v -q --branch "$(cat ~/cudf/RAPIDS_BRANCH)" --clone-upstream --depth 1 --single-branch --shallow-submodules;
560+
if [ "$PYTHON_PACKAGE_MANAGER" = "pip" ]; then rapids-make-pip-env --force; elif [ "$PYTHON_PACKAGE_MANAGER" = "conda" ]; then rapids-make-conda-env --force; fi;
561+
rapids-generate-scripts;
562+
build-all -j0 -DBUILD_BENCHMARKS=OFF -DBUILD_NUMA_SUPPORT=OFF -DBUILD_TESTS=OFF -DCUDF_BUILD_TESTUTIL=OFF --verbose 2>&1 | tee telemetry-artifacts/build.log;
556563
sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt;
557564
unit-tests-cudf-pandas:
558565
needs: [wheel-build-cudf, changed-files]

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ repos:
136136
name: cudf-polars-ir-signatures
137137
description: 'Validate cudf-polars IR.do_evaluate signatures.'
138138
entry: ./ci/check_cudf_polars_ir.py
139-
language: python
139+
language: script
140140
files: ^python/cudf_polars/cudf_polars/(dsl/ir|experimental/(shuffle|io|sort))\.py$
141141
pass_filenames: true
142142
verbose: true

‎ci/test_cudf_polars_experimental.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ rapids-pip-retry install \
2828
-v \
2929
--prefer-binary \
3030
--constraint "${PIP_CONSTRAINT}" \
31-
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental,rapidsmpf]" \
31+
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental]" \
3232
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
3333
"$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)"
3434

‎ci/test_cudf_polars_polars_tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ rapids-logger "Install libcudf, pylibcudf and cudf_polars"
2727
rapids-pip-retry install \
2828
-v \
2929
--constraint "${PIP_CONSTRAINT}" \
30-
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,rapidsmpf]" \
30+
"$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test]" \
3131
"$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
3232
"$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)"
3333

0 commit comments

Comments
 (0)