feat(libcuvs): port libcuvs to rattler-build

.gitignore

@@ -81,7 +81,8 @@ ivf_flat_index
 ivf_pq_index
 
 # cuvs_bench
-datasets/
+#datasets/
+!python/cuvs_bench/cuvs_bench/config/datasets/datasets.yaml
 /*.json
 
 # java

build.sh

@@ -18,7 +18,7 @@ ARGS=$*
 # scripts, and that this script resides in the repo dir!
 REPODIR=$(cd $(dirname $0); pwd)
 
-VALIDARGS="clean libcuvs python rust go java docs tests bench-ann examples --uninstall  -v -g -n --compile-static-lib --allgpuarch --no-mg --no-cpu --cpu-only --no-shared-libs --no-nvtx --show_depr_warn --incl-cache-stats --time -h"
+VALIDARGS="clean libcuvs python rust go java docs tests bench-ann examples --uninstall  -v -g -n --allgpuarch --no-mg --no-cpu --cpu-only --no-shared-libs --no-nvtx --show_depr_warn --incl-cache-stats --time -h"
 HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<tool>] [--limit-tests=<targets>] [--limit-bench-ann=<targets>] [--build-metrics=<filename>]
  where <target> is:
    clean            - remove all existing build artifacts and configuration (start over)
@@ -38,7 +38,6 @@ HELP="$0 [<target> ...] [<flag> ...] [--cmake-args=\"<args>\"] [--cache-tool=<to
    -g                          - build for debug
    -n                          - no install step
    --uninstall                 - uninstall files for specified targets which were built and installed prior
-   --compile-static-lib        - compile static library for all components
    --cpu-only                  - build CPU only components without CUDA. Currently only applies to bench-ann.
    --limit-tests               - semicolon-separated list of test executables to compile (e.g. NEIGHBORS_TEST;CLUSTER_TEST)
    --limit-bench-ann           - semicolon-separated list of ann benchmark executables to compute (e.g. HNSWLIB_ANN_BENCH;RAFT_IVF_PQ_ANN_BENCH)
@@ -332,7 +331,7 @@ fi
 
 ################################################################################
 # Configure for building all C++ targets
-if (( ${NUMARGS} == 0 )) || hasArg libcuvs || hasArg docs || hasArg tests || hasArg bench-prims || hasArg bench-ann; then
+if (( NUMARGS == 0 )) || hasArg libcuvs || hasArg docs || hasArg tests || hasArg bench-prims || hasArg bench-ann; then
     COMPILE_LIBRARY=ON
     if (( ${BUILD_SHARED_LIBS} == "OFF" )); then
         CMAKE_TARGET="${CMAKE_TARGET};"
@@ -437,7 +436,7 @@ if (( ${NUMARGS} == 0 )) || hasArg python; then
 fi
 
 # Build and (optionally) install the cuvs-bench Python package
-if (( ${NUMARGS} == 0 )) || hasArg bench-ann; then
+if (( NUMARGS == 0 )) || (hasArg bench-ann && ! hasArg -n); then
     python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ${REPODIR}/python/cuvs_bench
 fi
 

ci/build_cpp.sh

@@ -17,9 +17,25 @@ rapids-logger "Begin cpp build"
 
 sccache --zero-stats
 
-RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry build \
-  conda/recipes/libcuvs
+
+RAPIDS_PACKAGE_VERSION=$(rapids-generate-version)
+export RAPIDS_PACKAGE_VERSION
+
+source rapids-rattler-channel-string
+
+# --no-build-id allows for caching with `sccache`
+# more info is available at
+# https://rattler.build/latest/tips_and_tricks/#using-sccache-or-ccache-with-rattler-build
+rattler-build build --recipe conda/recipes/libcuvs \
+                    --experimental \
+                    --no-build-id \
+                    --channel-priority disabled \
+                    --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \
+                    "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
 
+# remove build_cache directory
+rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache
+
 rapids-upload-conda-to-s3 cpp

ci/build_python.sh

@@ -23,22 +23,39 @@ echo "${version}" > VERSION
 
 sccache --zero-stats
 
-# TODO: Remove `--no-test` flags once importing on a CPU
-# node works correctly
-rapids-conda-retry build \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  conda/recipes/cuvs
+
+# populates `RATTLER_CHANNELS` array
+source rapids-rattler-channel-string
+
+rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS"
+
+RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}")
+
+sccache --zero-stats
+
+rapids-logger "Building cuvs"
+
+# --no-build-id allows for caching with `sccache`
+# more info is available at
+# https://rattler.build/latest/tips_and_tricks/#using-sccache-or-ccache-with-rattler-build
+rattler-build build --recipe conda/recipes/cuvs \
+                    --experimental \
+                    --no-build-id \
+                    --channel-priority disabled \
+                    --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \
+                    --test skip \
+                    "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
 sccache --zero-stats
 
-# Build cuvs-bench for each cuda and python version
-rapids-conda-retry build \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  conda/recipes/cuvs-bench
+rattler-build build --recipe conda/recipes/cuvs-bench \
+                    --experimental \
+                    --no-build-id \
+                    --channel-priority disabled \
+                    --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \
+                    --test skip \
+                    "${RATTLER_CHANNELS[@]}"
 
 sccache --show-adv-stats
 sccache --zero-stats
@@ -47,11 +64,12 @@ sccache --zero-stats
 # version
 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
 if [[ ${RAPIDS_CUDA_MAJOR} == "12" ]]; then
-  rapids-conda-retry build \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  conda/recipes/cuvs-bench-cpu
-
+  rattler-build build --recipe conda/recipes/cuvs-bench-cpu \
+                      --experimental \
+                      --no-build-id \
+                      --channel-priority disabled \
+                      --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \
+                      "${RATTLER_CHANNELS[@]}"
   sccache --show-adv-stats
 fi
 

conda/recipes/cuvs-bench-cpu/recipe.yaml

@@ -0,0 +1,102 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.
+schema_version: 1
+
+context:
+  version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }}
+  date_string: '${{ env.get("RAPIDS_DATE_STRING") }}'
+  py_version: ${{ env.get("RAPIDS_PY_VERSION") }}
+  py_buildstring: ${{ py_version | version_to_buildstring }}
+  head_rev: ${{ git.head_rev(".")[:8] }}
+  linux64: ${{ linux and x86_64 }}
+
+package:
+  name: cuvs-bench-cpu
+  version: ${{ version }}
+
+source:
+  path: ../../..
+
+build:
+  string: py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }}
+  script:
+    content: |
+
+      # Remove `-fdebug-prefix-map` line from CFLAGS and CXXFLAGS so the
+      # incrementing version number in the compile line doesn't break the
+      # cache
+      set -x
+      export CFLAGS=$(echo $CFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g')
+      export CXXFLAGS=$(echo $CXXFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g')
+      set +x
+
+      ./build.sh bench-ann --cpu-only --no-nvtx --build-metrics=bench_ann_cpu --incl-cache-stats
+      cmake --install cpp/build --component ann_bench
+
+    secrets:
+      - AWS_ACCESS_KEY_ID
+      - AWS_SECRET_ACCESS_KEY
+      - AWS_SESSION_TOKEN
+    env:
+      CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }}
+      CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }}
+      CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }}
+      PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }}
+      RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }}
+      SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }}
+      SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }}
+      SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }}
+      SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }}
+      SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }}
+      SCCACHE_S3_KEY_PREFIX: libcuvs-${{ env.get("RAPIDS_CONDA_ARCH") }}
+
+requirements:
+  build:
+    - ${{ compiler("c") }}
+    - ${{ compiler("cxx") }}
+    - cmake ${{ cmake_version }}
+    - ninja
+    - ${{ stdlib("c") }}
+  host:
+    - benchmark
+    - glog ${{ glog_version }}
+    - nlohmann_json ${{ nlohmann_json_version }}
+    - openblas
+    - pip
+    - python =${{ py_version }}
+    - rapids-build-backend>=0.3.0,<0.4.0.dev0
+    - if: linux64
+      then:
+        - libaio
+        - libboost-devel=1.87
+        - mkl-devel=2023
+  run:
+    - benchmark
+    - click
+    - glog ${{ glog_version }}
+    - h5py ${{ h5py_version }}
+    - matplotlib-base
+    - numpy >=1.23,<3.0a0
+    - pandas
+    - pyyaml
+    - python
+    - requests
+    - scikit-learn>=1.5
+    - if: linux64
+      then:
+        - mkl =2023
+
+tests:
+  - python:
+      imports:
+        - cuvs_bench
+        - cuvs_bench.generate_groundtruth
+        - cuvs_bench.get_dataset
+        - cuvs_bench.plot
+        - cuvs_bench.run
+        - cuvs_bench.split_groundtruth
+      pip_check: false
+
+about:
+  homepage: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.urls.Homepage }}
+  license: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.license.text }}
+  summary: cuVS CPU benchmark