Skip to content

Commit b882590

Browse files
tarang-jaintarang-jain
authored
Enable building FAISS main statically (#241)
Port rapidsai/raft#2323 PR from RAFT [Cleans up a collection of anti-patterns in the cuvs CMake code while also enabling building faiss from latest main] Authors: - Tarang Jain (https://github.com/tarang-jain) Approvers: - Robert Maynard (https://github.com/robertmaynard) - Corey J. Nolet (https://github.com/cjnolet) - Paul Taylor (https://github.com/trxcllnt) - Ray Douglass (https://github.com/raydouglass) URL: #241
1 parent 337c696 commit b882590

10 files changed

Lines changed: 286 additions & 267 deletions

File tree

‎.devcontainer/Dockerfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,13 @@ ARG PYTHON_PACKAGE_MANAGER=conda
55

66
FROM ${BASE} as pip-base
77

8+
RUN apt update -y \
9+
&& DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \
10+
# faiss dependencies
11+
libblas-dev \
12+
liblapack-dev \
13+
&& rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*;
14+
815
ENV DEFAULT_VIRTUAL_ENV=rapids
916

1017
FROM ${BASE} as conda-base

‎cpp/CMakeLists.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -682,13 +682,13 @@ rapids_export(
682682
# * build test executable ----------------------------------------------------
683683

684684
if(BUILD_TESTS OR BUILD_C_TESTS)
685-
include(internal/CMakeLists.txt)
686-
include(test/CMakeLists.txt)
685+
add_subdirectory(internal)
686+
add_subdirectory(test)
687687
endif()
688688

689689
# ##################################################################################################
690690
# * build ann benchmark executable -----------------------------------------------
691691

692692
if(BUILD_ANN_BENCH)
693-
include(bench/ann/CMakeLists.txt)
693+
add_subdirectory(bench/ann/)
694694
endif()

‎cpp/bench/ann/CMakeLists.txt‎

Lines changed: 42 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
# the License.
1313
# =============================================================================
1414

15+
list(APPEND CMAKE_MODULE_PATH "${CUVS_SOURCE_DIR}")
16+
1517
# ##################################################################################################
1618
# * benchmark options ------------------------------------------------------------------------------
1719

@@ -39,31 +41,18 @@ option(CUVS_ANN_BENCH_SINGLE_EXE
3941

4042
find_package(Threads REQUIRED)
4143

44+
set(CUVS_ANN_BENCH_USE_FAISS ON)
45+
set(CUVS_FAISS_ENABLE_GPU ON)
46+
set(CUVS_USE_FAISS_STATIC ON)
47+
4248
if(BUILD_CPU_ONLY)
4349
set(CUVS_FAISS_ENABLE_GPU OFF)
44-
set(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT OFF)
45-
set(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT OFF)
46-
set(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ OFF)
4750
set(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT OFF)
4851
set(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ OFF)
4952
set(CUVS_ANN_BENCH_USE_CUVS_CAGRA OFF)
5053
set(CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE OFF)
5154
set(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB OFF)
5255
set(CUVS_ANN_BENCH_USE_GGNN OFF)
53-
else()
54-
set(CUVS_FAISS_ENABLE_GPU ON)
55-
endif()
56-
57-
set(CUVS_ANN_BENCH_USE_FAISS OFF)
58-
if(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT
59-
OR CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ
60-
OR CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT
61-
OR CUVS_ANN_BENCH_USE_FAISS_CPU_FLAT
62-
OR CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_PQ
63-
OR CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT
64-
)
65-
set(CUVS_ANN_BENCH_USE_FAISS ON)
66-
set(CUVS_USE_FAISS_STATIC ON)
6756
endif()
6857

6958
set(CUVS_ANN_BENCH_USE_CUVS OFF)
@@ -80,21 +69,17 @@ endif()
8069
# * Fetch requirements -------------------------------------------------------------
8170

8271
if(CUVS_ANN_BENCH_USE_HNSWLIB OR CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB)
83-
include(cmake/thirdparty/get_hnswlib.cmake)
72+
include(cmake/thirdparty/get_hnswlib)
8473
endif()
8574

86-
include(cmake/thirdparty/get_nlohmann_json.cmake)
75+
include(cmake/thirdparty/get_nlohmann_json)
8776

8877
if(CUVS_ANN_BENCH_USE_GGNN)
89-
include(cmake/thirdparty/get_ggnn.cmake)
78+
include(cmake/thirdparty/get_ggnn)
9079
endif()
9180

9281
if(CUVS_ANN_BENCH_USE_FAISS)
93-
# We need to ensure that faiss has all the conda information. So we currently use the very ugly
94-
# hammer of `link_libraries` to ensure that all targets in this directory and the faiss directory
95-
# will have the conda includes/link dirs
96-
link_libraries($<TARGET_NAME_IF_EXISTS:conda_env>)
97-
include(cmake/thirdparty/get_faiss.cmake)
82+
include(cmake/thirdparty/get_faiss)
9883
endif()
9984

10085
# ##################################################################################################
@@ -154,8 +139,6 @@ function(ConfigureAnnBench)
154139
$<$<BOOL:${GPU_BUILD}>:CUDA::cudart_static>
155140
$<TARGET_NAME_IF_EXISTS:OpenMP::OpenMP_CXX>
156141
$<TARGET_NAME_IF_EXISTS:conda_env>
157-
-static-libgcc
158-
-static-libstdc++
159142
)
160143

161144
set_target_properties(
@@ -210,131 +193,123 @@ endif()
210193

211194
if(CUVS_ANN_BENCH_USE_HNSWLIB)
212195
ConfigureAnnBench(
213-
NAME HNSWLIB PATH bench/ann/src/hnswlib/hnswlib_benchmark.cpp LINKS hnswlib::hnswlib
196+
NAME HNSWLIB PATH src/hnswlib/hnswlib_benchmark.cpp LINKS hnswlib::hnswlib
214197
)
215198

216199
endif()
217200

218201
if(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ)
219202
ConfigureAnnBench(
220-
NAME CUVS_IVF_PQ PATH bench/ann/src/cuvs/cuvs_benchmark.cu
221-
$<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_IVF_PQ}>:bench/ann/src/cuvs/cuvs_ivf_pq.cu> LINKS cuvs
203+
NAME CUVS_IVF_PQ
204+
PATH
205+
src/cuvs/cuvs_benchmark.cu
206+
src/cuvs/cuvs_ivf_pq.cu
207+
LINKS cuvs
222208
)
223209
endif()
224210

225211
if(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT)
226212
ConfigureAnnBench(
227-
NAME CUVS_IVF_FLAT PATH bench/ann/src/cuvs/cuvs_benchmark.cu
228-
$<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT}>:bench/ann/src/cuvs/cuvs_ivf_flat.cu> LINKS cuvs
213+
NAME CUVS_IVF_FLAT
214+
PATH
215+
src/cuvs/cuvs_benchmark.cu
216+
src/cuvs/cuvs_ivf_flat.cu
217+
LINKS
218+
cuvs
229219
)
230220
endif()
231221

232222
if(CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE)
233-
ConfigureAnnBench(NAME CUVS_BRUTE_FORCE PATH bench/ann/src/cuvs/cuvs_benchmark.cu LINKS cuvs)
223+
ConfigureAnnBench(NAME CUVS_BRUTE_FORCE PATH src/cuvs/cuvs_benchmark.cu LINKS cuvs)
234224
endif()
235225

236226
if(CUVS_ANN_BENCH_USE_CUVS_CAGRA)
237227
ConfigureAnnBench(
238228
NAME
239229
CUVS_CAGRA
240230
PATH
241-
bench/ann/src/cuvs/cuvs_benchmark.cu
242-
$<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_CAGRA}>:bench/ann/src/cuvs/cuvs_cagra_float.cu>
243-
$<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_CAGRA}>:bench/ann/src/cuvs/cuvs_cagra_half.cu>
244-
$<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_CAGRA}>:bench/ann/src/cuvs/cuvs_cagra_int8_t.cu>
245-
$<$<BOOL:${CUVS_ANN_BENCH_USE_CUVS_CAGRA}>:bench/ann/src/cuvs/cuvs_cagra_uint8_t.cu>
231+
src/cuvs/cuvs_benchmark.cu
232+
src/cuvs/cuvs_cagra_float.cu
233+
src/cuvs/cuvs_cagra_half.cu
234+
src/cuvs/cuvs_cagra_int8_t.cu
235+
src/cuvs/cuvs_cagra_uint8_t.cu
246236
LINKS
247237
cuvs
248238
)
249239
endif()
250240

251241
if(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB)
252242
ConfigureAnnBench(
253-
NAME CUVS_CAGRA_HNSWLIB PATH bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu LINKS cuvs
243+
NAME CUVS_CAGRA_HNSWLIB PATH src/cuvs/cuvs_cagra_hnswlib.cu LINKS cuvs
254244
hnswlib::hnswlib
255245
)
256246
endif()
257247

258-
set(CUVS_FAISS_TARGETS faiss::faiss)
259-
if(TARGET faiss::faiss_avx2)
260-
set(CUVS_FAISS_TARGETS faiss::faiss_avx2)
261-
endif()
262-
263248
message("CUVS_FAISS_TARGETS: ${CUVS_FAISS_TARGETS}")
264249
message("CUDAToolkit_LIBRARY_DIR: ${CUDAToolkit_LIBRARY_DIR}")
265250
if(CUVS_ANN_BENCH_USE_FAISS_CPU_FLAT)
266251
ConfigureAnnBench(
267-
NAME FAISS_CPU_FLAT PATH bench/ann/src/faiss/faiss_cpu_benchmark.cpp LINKS
252+
NAME FAISS_CPU_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS
268253
${CUVS_FAISS_TARGETS}
269254
)
270255
endif()
271256

272257
if(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT)
273258
ConfigureAnnBench(
274-
NAME FAISS_CPU_IVF_FLAT PATH bench/ann/src/faiss/faiss_cpu_benchmark.cpp LINKS
259+
NAME FAISS_CPU_IVF_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS
275260
${CUVS_FAISS_TARGETS}
276261
)
277262
endif()
278263

279264
if(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_PQ)
280265
ConfigureAnnBench(
281-
NAME FAISS_CPU_IVF_PQ PATH bench/ann/src/faiss/faiss_cpu_benchmark.cpp LINKS
266+
NAME FAISS_CPU_IVF_PQ PATH src/faiss/faiss_cpu_benchmark.cpp LINKS
282267
${CUVS_FAISS_TARGETS}
283268
)
284269
endif()
285270

286-
if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT)
271+
if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT AND CUVS_FAISS_ENABLE_GPU)
287272
ConfigureAnnBench(
288-
NAME FAISS_GPU_IVF_FLAT PATH bench/ann/src/faiss/faiss_gpu_benchmark.cu LINKS
273+
NAME FAISS_GPU_IVF_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS
289274
${CUVS_FAISS_TARGETS}
290275
)
291276
endif()
292277

293-
if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ)
278+
if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ AND CUVS_FAISS_ENABLE_GPU)
294279
ConfigureAnnBench(
295-
NAME FAISS_GPU_IVF_PQ PATH bench/ann/src/faiss/faiss_gpu_benchmark.cu LINKS
280+
NAME FAISS_GPU_IVF_PQ PATH src/faiss/faiss_gpu_benchmark.cu LINKS
296281
${CUVS_FAISS_TARGETS}
297282
)
298283
endif()
299284

300-
if(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT)
285+
if(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT AND CUVS_FAISS_ENABLE_GPU)
301286
ConfigureAnnBench(
302-
NAME FAISS_GPU_FLAT PATH bench/ann/src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS}
287+
NAME FAISS_GPU_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS}
303288
)
304289
endif()
305290

306291
if(CUVS_ANN_BENCH_USE_GGNN)
307-
include(cmake/thirdparty/get_glog.cmake)
292+
include(cmake/thirdparty/get_glog)
308293
ConfigureAnnBench(
309-
NAME GGNN PATH bench/ann/src/ggnn/ggnn_benchmark.cu LINKS glog::glog ggnn::ggnn CUDA::curand
294+
NAME GGNN PATH src/ggnn/ggnn_benchmark.cu LINKS glog::glog ggnn::ggnn CUDA::curand
310295
)
311296
endif()
312297

313298
# ##################################################################################################
314299
# * Dynamically-loading ANN_BENCH executable -------------------------------------------------------
315300
if(CUVS_ANN_BENCH_SINGLE_EXE)
316-
add_executable(ANN_BENCH bench/ann/src/common/benchmark.cpp)
317-
318-
# Build and link static version of the GBench to keep ANN_BENCH self-contained.
319-
get_target_property(TMP_PROP benchmark::benchmark SOURCES)
320-
add_library(benchmark_static STATIC ${TMP_PROP})
321-
get_target_property(TMP_PROP benchmark::benchmark INCLUDE_DIRECTORIES)
322-
target_include_directories(benchmark_static PUBLIC ${TMP_PROP})
323-
get_target_property(TMP_PROP benchmark::benchmark LINK_LIBRARIES)
324-
target_link_libraries(benchmark_static PUBLIC ${TMP_PROP})
301+
add_executable(ANN_BENCH src/common/benchmark.cpp)
325302

326303
target_include_directories(ANN_BENCH PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
327304

328305
target_link_libraries(
329306
ANN_BENCH
330307
PRIVATE raft::raft
331308
nlohmann_json::nlohmann_json
332-
benchmark_static
309+
benchmark::benchmark
333310
dl
334-
-static-libgcc
335311
fmt::fmt-header-only
336312
spdlog::spdlog_header_only
337-
-static-libstdc++
338313
$<$<BOOL:${NVTX3_HEADERS_FOUND}>:CUDA::nvtx3>
339314
)
340315
set_target_properties(

‎cpp/cmake/patches/faiss_override.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"packages" : {
3+
"faiss" : {
4+
"version": "1.7.4",
5+
"git_url": "https://github.com/facebookresearch/faiss.git",
6+
"git_tag": "main"
7+
}
8+
}
9+
}

‎cpp/cmake/patches/ggnn_override.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"packages" : {
3+
"ggnn" : {
4+
"version": "0.5",
5+
"git_url": "https://github.com/cgtuebingen/ggnn.git",
6+
"git_tag": "release_${version}",
7+
"patches" : [
8+
{
9+
"file" : "${current_json_dir}/ggnn.diff",
10+
"issue" : "Correct compilation issues",
11+
"fixed_in" : ""
12+
}
13+
]
14+
}
15+
}
16+
}

‎cpp/cmake/patches/hnswlib_override.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"packages" : {
3+
"hnswlib" : {
4+
"version": "0.6.2",
5+
"git_url": "https://github.com/nmslib/hnswlib.git",
6+
"git_tag": "v${version}",
7+
"patches" : [
8+
{
9+
"file" : "${current_json_dir}/hnswlib.diff",
10+
"issue" : "Correct compilation issues",
11+
"fixed_in" : ""
12+
}
13+
]
14+
}
15+
}
16+
}

0 commit comments

Comments
 (0)