Prepare cuvs for removal of deprecated raft apis (#1610)

Resolves #1601, Resolves #1491

What does this PR do?

1. Removes lingering **unused** raft headers that will be deprecated such as `#include <raft/neighbors/refine.cuh>`, `#include <raft/distance/distance_types.hpp>`, etc.
2. Exposes `build_dendrogram_host` which is used by cuml. cuml currently uses the raft version. The raft version was moved to cuvs but it is not in the public interface.
3. This PR also removes the legacy spectral related files.

Authors:
  - Anupam (https://github.com/aamijar)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Jinsol Park (https://github.com/jinsolp)
  - Divye Gala (https://github.com/divyegala)

URL: #1610

Author: aamijar

cpp/CMakeLists.txt

@@ -338,6 +338,7 @@ if(NOT BUILD_CPU_ONLY)
   add_library(
     cuvs_objs OBJECT
     src/cluster/detail/minClusterDistanceCompute.cu
+    src/cluster/agglomerative.cu
     src/cluster/kmeans_cluster_cost.cu
     src/cluster/kmeans_fit_mg_float.cu
     src/cluster/kmeans_fit_mg_double.cu
@@ -414,7 +415,6 @@ if(NOT BUILD_CPU_ONLY)
     src/distance/distance.cu
     src/distance/pairwise_distance.cu
     src/distance/sparse_distance.cu
-    src/embed/spectral.cu
     src/neighbors/all_neighbors/all_neighbors.cu
     src/neighbors/ball_cover.cu
     src/neighbors/epsilon_neighborhood.cu

cpp/bench/ann/src/cuvs/cuvs_ivf_pq_wrapper.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2023-2024, NVIDIA CORPORATION.
+ * SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION.
  * SPDX-License-Identifier: Apache-2.0
  */
 #pragma once
@@ -19,7 +19,6 @@
 #include <raft/core/logger.hpp>
 #include <raft/core/resource/cuda_stream.hpp>
 #include <raft/linalg/unary_op.cuh>
-#include <raft/neighbors/refine.cuh>
 #include <raft/util/cudart_utils.hpp>
 #include <rmm/cuda_stream_pool.hpp>
 

cpp/include/cuvs/cluster/agglomerative.hpp

@@ -227,6 +227,32 @@ void build_linkage(
   raft::device_vector_view<float, int64_t> out_distances,
   raft::device_vector_view<int64_t, int64_t> out_sizes,
   std::optional<raft::device_vector_view<float, int64_t>> core_dists);
+
+/**
+ * Build dendrogram from a Minimum Spanning Tree (MST).
+ *
+ * This function takes a sorted MST (represented as edges with source, destination, and weights)
+ * and constructs a dendrogram (hierarchical clustering tree) on the host.
+ *
+ * @param[in] handle The raft resources handle
+ * @param[in] rows Source nodes of the MST edges (device memory, size: nnz)
+ * @param[in] cols Destination nodes of the MST edges (device memory, size: nnz)
+ * @param[in] data Edge weights/distances of the MST (device memory, size: nnz)
+ * @param[out] children Output dendrogram children array (device memory, size: nnz * 2)
+ *                      Each pair of consecutive elements represents the two children
+ *                      merged at each step of the hierarchy
+ * @param[out] out_delta Output distances/heights at which clusters are merged (device memory, size:
+ * nnz)
+ * @param[out] out_size Output cluster sizes at each merge step (device memory, size: nnz)
+ */
+void build_dendrogram(raft::resources const& handle,
+                      raft::device_vector_view<const int64_t, int64_t> rows,
+                      raft::device_vector_view<const int64_t, int64_t> cols,
+                      raft::device_vector_view<const float, int64_t> data,
+                      raft::device_matrix_view<int64_t, int64_t, raft::row_major> children,
+                      raft::device_vector_view<float, int64_t> out_delta,
+                      raft::device_vector_view<int64_t, int64_t> out_size);
+
 }  // namespace helpers
 /**
  * @}

cpp/include/cuvs/embed/spectral.hpp

@@ -0,0 +1,36 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include "./detail/agglomerative.cuh"
+
+#include <cuvs/cluster/agglomerative.hpp>
+
+namespace cuvs::cluster::agglomerative::helpers {
+
+#define CUVS_INST_AGGLOMERATIVE(IdxT, ValueT)                                           \
+  void build_dendrogram(raft::resources const& handle,                                  \
+                        raft::device_vector_view<const IdxT, IdxT> rows,                \
+                        raft::device_vector_view<const IdxT, IdxT> cols,                \
+                        raft::device_vector_view<const ValueT, IdxT> data,              \
+                        raft::device_matrix_view<IdxT, IdxT, raft::row_major> children, \
+                        raft::device_vector_view<ValueT, IdxT> out_delta,               \
+                        raft::device_vector_view<IdxT, IdxT> out_size)                  \
+  {                                                                                     \
+    size_t nnz = rows.extent(0);                                                        \
+    detail::build_dendrogram_host<IdxT, ValueT>(handle,                                 \
+                                                rows.data_handle(),                     \
+                                                cols.data_handle(),                     \
+                                                data.data_handle(),                     \
+                                                nnz,                                    \
+                                                children.data_handle(),                 \
+                                                out_delta.data_handle(),                \
+                                                out_size.data_handle());                \
+  }
+
+CUVS_INST_AGGLOMERATIVE(int64_t, float);
+
+#undef CUVS_INST_AGGLOMERATIVE
+
+}  // namespace cuvs::cluster::agglomerative::helpers

cpp/src/cluster/agglomerative.cu

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION.
+ * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION.
  * SPDX-License-Identifier: Apache-2.0
  */
 
@@ -10,7 +10,6 @@
 #include <raft/core/operators.cuh>
 #include <raft/core/operators.hpp>
 #include <raft/core/resource/cuda_stream.hpp>
-#include <raft/distance/distance_types.hpp>
 #include <raft/sparse/convert/coo.cuh>
 #include <raft/sparse/csr.hpp>
 #include <raft/sparse/detail/utils.h>

cpp/src/distance/detail/sparse/lp_distance.cuh

@@ -27,9 +27,9 @@ namespace cuvs::neighbors::ball_cover::detail {
  * @code{.cpp}
  *
  *  #include <raft/core/resources.hpp>
- *  #include <raft/neighbors/ball_cover.cuh>
- *  #include <raft/distance/distance_types.hpp>
- *  using namespace raft::neighbors;
+ *  #include <cuvs/neighbors/ball_cover.cuh>
+ *  #include <cuvs/distance/distance.hpp>
+ *  using namespace cuvs::neighbors;
  *
  *  raft::resources handle;
  *  ...
@@ -359,9 +359,9 @@ void eps_nn(raft::resources const& handle,
  * @code{.cpp}
  *
  *  #include <raft/core/resources.hpp>
- *  #include <raft/neighbors/ball_cover.cuh>
- *  #include <raft/distance/distance_types.hpp>
- *  using namespace raft::neighbors;
+ *  #include <cuvs/neighbors/ball_cover.cuh>
+ *  #include <cuvs/distance/distance.hpp>
+ *  using namespace cuvs::neighbors;
  *
  *  raft::resources handle;
  *  ...

cpp/src/embed/spectral.cu

@@ -9,7 +9,6 @@
 #include <raft/core/mdspan_types.hpp>
 #include <raft/core/resource/cuda_stream.hpp>
 #include <raft/core/resources.hpp>
-#include <raft/spatial/knn/detail/ann_utils.cuh>
 #include <raft/stats/histogram.cuh>
 
 #include <rmm/device_buffer.hpp>

cpp/src/neighbors/ball_cover.cuh

@@ -31,7 +31,6 @@
 #include <raft/matrix/init.cuh>
 #include <raft/sparse/convert/coo.cuh>
 #include <raft/sparse/convert/csr.cuh>
-#include <raft/sparse/distance/detail/utils.cuh>
 #include <raft/sparse/linalg/masked_matmul.hpp>
 #include <raft/sparse/matrix/select_k.cuh>
 #include <raft/util/cuda_utils.cuh>