diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2c7a61fd39..ce265d7163 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -9,266 +9,28 @@ concurrency: jobs: pr-builder: needs: - - check-nightly-ci - - changed-files - - checks - - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - - conda-python-build - - conda-python-tests - - conda-java-tests - - docs-build - - rust-build - - go-build - - wheel-build-libcuvs - - wheel-build-cuvs - - wheel-tests-cuvs - - devcontainer - - telemetry-setup + - java-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda13.0 if: always() with: needs: ${{ toJSON(needs) }} - telemetry-setup: - continue-on-error: true - runs-on: ubuntu-latest - env: - OTEL_SERVICE_NAME: 'pr-cuvs' - steps: - - name: Telemetry setup - if: ${{ vars.TELEMETRY_ENABLED == 'true' }} - uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main - check-nightly-ci: - needs: telemetry-setup - runs-on: ubuntu-latest - env: - RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: Check if nightly CI is passing - uses: rapidsai/shared-actions/check_nightly_success/dispatch@main - with: - repo: cuvs - max_days_without_success: 30 - changed-files: - needs: telemetry-setup - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda13.0 - with: - files_yaml: | - test_cpp: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!ci/release/update-version.sh' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!python/**' - - '!rust/**' - - '!go/**' - - '!thirdparty/LICENSES/**' - test_java: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!python/**' - - '!rust/**' - - '!go/**' - - '!thirdparty/LICENSES/**' - test_notebooks: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!ci/release/update-version.sh' - - '!README.md' - - '!rust/**' - - '!go/**' - - '!thirdparty/LICENSES/**' - test_python: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!ci/release/update-version.sh' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!rust/**' - - '!go/**' - - '!thirdparty/LICENSES/**' - checks: - needs: telemetry-setup - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda13.0 - with: - enable_check_generated_files: false - ignored_pr_jobs: "telemetry-summarize" - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@cuda13.0 - with: - build_type: pull-request - node_type: cpu16 - script: ci/build_cpp.sh - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@cuda13.0 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - script: ci/test_cpp.sh - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@cuda13.0 - with: - build_type: pull-request - symbol_exclusions: (void (thrust::|cub::)) - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@cuda13.0 - with: - build_type: pull-request - script: ci/build_python.sh - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@cuda13.0 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_python.sh - conda-java-tests: - needs: [conda-cpp-build, changed-files] + java-build: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: fail-fast: false matrix: cuda_version: - '12.9.1' - - '13.0.0' + # - '13.0.0' with: build_type: pull-request - node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" - script: "ci/test_java.sh" + # ci-wheel Rocky Linux 8 images are used to ensure the built libcuvs{_c}.so and dependencies + # respect RAPIDS minimum GLIBC version and are portable for re-use in environments not using + # 'conda' or 'pip' + container_image: "rapidsai/ci-wheel:25.10-cuda${{ matrix.cuda_version }}-rockylinux8-py3.13" + script: "ci/build_java_portable_fat_jar.sh" artifact-name: "cuvs-java-cuda${{ matrix.cuda_version }}" file_to_upload: "java/cuvs-java/target/" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-latest" - script: "ci/build_docs.sh" - rust-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: - cuda_version: - - '12.9.1' - - '13.0.0' - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" - script: "ci/build_rust.sh" - go-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: - cuda_version: - - '12.9.1' - - '13.0.0' - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" - script: "ci/build_go.sh" - wheel-build-libcuvs: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda13.0 - with: - build_type: pull-request - script: ci/build_wheel_libcuvs.sh - # build for every combination of arch and CUDA version, but only for the latest Python - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) - package-name: libcuvs - package-type: cpp - wheel-build-cuvs: - needs: wheel-build-libcuvs - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@cuda13.0 - with: - build_type: pull-request - script: ci/build_wheel_cuvs.sh - package-name: cuvs - package-type: python - wheel-tests-cuvs: - needs: [wheel-build-cuvs, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@cuda13.0 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cuvs.sh - devcontainer: - secrets: inherit - needs: telemetry-setup - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@cuda13.0 - with: - arch: '["amd64", "arm64"]' - cuda: '["13.0"]' - node_type: "cpu8" - rapids-aux-secret-1: GIST_REPO_READ_ORG_GITHUB_TOKEN - env: | - SCCACHE_DIST_MAX_RETRIES=inf - SCCACHE_SERVER_LOG=sccache=debug - SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=false - SCCACHE_DIST_AUTH_TOKEN_VAR=RAPIDS_AUX_SECRET_1 - build_command: | - sccache --zero-stats; - build-all -j0 --verbose 2>&1 | tee telemetry-artifacts/build.log; - sccache --show-adv-stats | tee telemetry-artifacts/sccache-stats.txt; - telemetry-summarize: - # This job must use a self-hosted runner to record telemetry traces. - runs-on: linux-amd64-cpu4 - needs: pr-builder - if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} - continue-on-error: true - steps: - - name: Telemetry summarize - uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main - env: - GH_TOKEN: ${{ github.token }} diff --git a/.shellcheckrc b/.shellcheckrc index b57b9d1962..8ba826415c 100644 --- a/.shellcheckrc +++ b/.shellcheckrc @@ -1,2 +1,4 @@ # Disable file checks (otherwise every use of `gha-tools` will get flagged) disable=SC1091 +# Disable checks saying that 'pushd' / 'popd' should be followed by '|| exit' or similar in case they don't exist +disable=SC2164 diff --git a/ci/build_java_portable_fat_jar.sh b/ci/build_java_portable_fat_jar.sh new file mode 100755 index 0000000000..73b491f370 --- /dev/null +++ b/ci/build_java_portable_fat_jar.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Copyright (c) 2025, NVIDIA CORPORATION. + +# builds cuvs-java jar +# +# Intentionally does not use environment-management tools like 'conda', to ensure +# the built binaries are portable to other environments not using those tools. +# +set -e -u -o pipefail + +export CMAKE_VERSION='3.30.4' +export JDK_VERSION='22.0.2' +export MAVEN_VERSION='3.9.11' + +source rapids-configure-sccache + +export CMAKE_GENERATOR=Ninja + +rapids-print-env + +# TODO: Remove this argument-handling when build and test workflows are separated, +# and test_java.sh no longer calls build_java.sh +# ref: https://github.com/rapidsai/cuvs/issues/868 +EXTRA_BUILD_ARGS=() +if [[ "${1:-}" == "--run-java-tests" ]]; then + EXTRA_BUILD_ARGS+=("--run-java-tests") +fi + +rapids-logger "installing JDK ${JDK_VERSION}" +wget \ + --quiet \ + -O /usr/local/jdk.tar.gz \ + "https://download.oracle.com/java/22/archive/jdk-${JDK_VERSION}_linux-x64_bin.tar.gz" + +pushd /usr/local + tar -xzf ./jdk.tar.gz + rm -rf ./jdk.tar.gz + JAVA_HOME="$(pwd)/jdk-${JDK_VERSION}" + export JAVA_HOME + PATH="${JAVA_HOME}/bin:${PATH}" + export PATH +popd + +# update to newer cmake +rapids-logger "install CMake ${CMAKE_VERSION}" +wget \ + --quiet \ + -O /usr/local/cmake.tar.gz \ + "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz" + +pushd /usr/local +tar -xzf ./cmake.tar.gz +rm -rf ./cmake.tar.gz +PATH="$(pwd)/cmake-${CMAKE_VERSION}-linux-x86_64/bin:${PATH}" +export PATH +popd + +# install newer Maven +rapids-logger "installing Maven ${MAVEN_VERSION}" +wget \ + --quiet \ + -O /usr/local/maven.tar.gz \ + "https://dlcdn.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz" + +pushd /usr/local +tar -xzf ./maven.tar.gz +rm -rf ./maven.tar.gz +M2_HOME="$(pwd)/apache-maven-${MAVEN_VERSION}" +export M2_HOME +PATH="${M2_HOME}/bin:${PATH}" +export PATH +popd + +# install other build tools that don't come pre-installed +dnf install -y \ + ninja-build + +# build libcuvs.so and cuvs-java +./build.sh \ + libcuvs \ + java \ + "${EXTRA_BUILD_ARGS[@]}" + +# check the produced artifacts +echo "found the following jars:" +find . -type f -name '*.jar' \ + -exec bash -c 'echo ""; echo "$1"; echo ""; jar tf $1' {} \+ + +echo "" +echo "checking dependencies: libcuvs.so" +echo "" +echo "--- ldd ---" +ldd -v ./cpp/build/libcuvs.so + +echo "--- readelf ---" +readelf -d ./ + +echo "" +echo "checking dependencies: libcuvs_c.so" +echo "" +echo "--- ldd ---" +ldd -v ./cpp/build/libcuvs_c.so + +echo "--- readelf ---" +readelf -d ./cpp/build/libcuvs_c.so diff --git a/ci/test_java.sh b/ci/test_java.sh index 20c19b6dee..2c3a7fc97a 100755 --- a/ci/test_java.sh +++ b/ci/test_java.sh @@ -17,7 +17,7 @@ export RAPIDS_CUDA_MAJOR # TODO: switch to installing pre-built artifacts instead of rebuilding in test jobs # ref: https://github.com/rapidsai/cuvs/issues/868 -ci/build_java.sh --run-java-tests +ci/build_java_portable_fat_jar.sh --run-java-tests rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE}