From 3fe4eb836f57b032ff79cd176c8d6e3a4e7ad797 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Mon, 15 Apr 2024 15:36:28 +0200 Subject: [PATCH 1/9] Added Faiss Building version v1.7.4 to get support for aarch64-linux-gnu - avoids ARM NEON optimization (which uses vld1q_f32_x2 etc.) included in v1.8.0: https://github.com/facebookresearch/faiss/pull/3166 --- F/Faiss/build_tarballs.jl | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 F/Faiss/build_tarballs.jl diff --git a/F/Faiss/build_tarballs.jl b/F/Faiss/build_tarballs.jl new file mode 100644 index 00000000000..53074f52f8d --- /dev/null +++ b/F/Faiss/build_tarballs.jl @@ -0,0 +1,75 @@ +# Note that this script can accept some limited command-line arguments, run +# `julia build_tarballs.jl --help` to see a usage message. +using BinaryBuilder, Pkg + +name = "Faiss" +version = v"1.7.4" + +# Collection of sources required to complete build +sources = [ + GitSource("https://github.com/facebookresearch/faiss.git", "d87888b13e7eb339bb9c45825e9d20def6665171"), +] + +# Bash recipe for building across all platforms +script = raw""" +# Needs CMake >= 3.23.1 provided via HostBuildDependency +apk del cmake + +cd faiss + +cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$prefix \ + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \ + -DCMAKE_BUILD_TYPE=Release \ + -DFAISS_ENABLE_GPU=OFF \ + -DFAISS_ENABLE_PYTHON=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DFAISS_ENABLE_C_API=ON +cmake --build build --parallel ${nproc} +cmake --install build + +install -Dvm 755 build/c_api/libfaiss_c.$dlext $libdir/libfaiss_c.$dlext +""" + +# These are the platforms we will build for by default, unless further +# platforms are passed in on the command line +platforms = supported_platforms() +filter!(p -> !(Sys.islinux(p) && libc(p) == "musl"), platforms) # musl builds fail - fixed in v1.8.0 +filter!(!Sys.isfreebsd, platforms) # freebsd builds fail - fixed in v1.8.0 +filter!(!Sys.iswindows, platforms) # Windows builds fail to link: undefined reference to `faiss::OnDiskInvertedListsIOHook::OnDiskInvertedListsIOHook()' + +mkl_platforms = Platform[ + Platform("x86_64", "Linux"), + Platform("i686", "Linux"), + Platform("x86_64", "MacOS"), + Platform("x86_64", "Windows"), +] + +openblas_platforms = filter(p -> p ∉ mkl_platforms, platforms) + +platforms = expand_cxxstring_abis(platforms) +mkl_platforms = expand_cxxstring_abis(mkl_platforms) +openblas_platforms = expand_cxxstring_abis(openblas_platforms) + +# The products that we will ensure are always built +products = Product[ + LibraryProduct(["libfaiss", "faiss"], :libfaiss), + LibraryProduct(["libfaiss_c", "faiss_c"], :libfaiss_c), +] + +# Dependencies that must be installed before this package can be built +dependencies = [ + # For OpenMP we use libomp from `LLVMOpenMP_jll` where we use LLVM as compiler (BSD + # systems), and libgomp from `CompilerSupportLibraries_jll` everywhere else. + Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"); platforms=filter(!Sys.isbsd, platforms)), + Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e"); platforms=filter(Sys.isbsd, platforms)), + Dependency("LAPACK_jll"; platforms = openblas_platforms), + Dependency("MKL_jll"; platforms = mkl_platforms), + BuildDependency("MKL_Headers_jll"; platforms = mkl_platforms), + Dependency("OpenBLAS32_jll"; platforms = openblas_platforms), + HostBuildDependency(PackageSpec("CMake_jll", v"3.23.3")), +] + +# Build the tarballs, and possibly a `build.jl` as well. +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"7") From 2be9b466d7f97d5d5bd54226158c87e9bd2709c8 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Mon, 15 Apr 2024 15:36:28 +0200 Subject: [PATCH 2/9] [Faiss] Updated to v1.8.0 --- F/Faiss/build_tarballs.jl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/F/Faiss/build_tarballs.jl b/F/Faiss/build_tarballs.jl index 53074f52f8d..f4ff24061e4 100644 --- a/F/Faiss/build_tarballs.jl +++ b/F/Faiss/build_tarballs.jl @@ -3,11 +3,11 @@ using BinaryBuilder, Pkg name = "Faiss" -version = v"1.7.4" +version = v"1.8.0" # Collection of sources required to complete build sources = [ - GitSource("https://github.com/facebookresearch/faiss.git", "d87888b13e7eb339bb9c45825e9d20def6665171"), + GitSource("https://github.com/facebookresearch/faiss.git", "22f2292be4aa98559797e1dd6e45f12386b06099") ] # Bash recipe for building across all platforms @@ -35,9 +35,6 @@ install -Dvm 755 build/c_api/libfaiss_c.$dlext $libdir/libfaiss_c.$dlext # These are the platforms we will build for by default, unless further # platforms are passed in on the command line platforms = supported_platforms() -filter!(p -> !(Sys.islinux(p) && libc(p) == "musl"), platforms) # musl builds fail - fixed in v1.8.0 -filter!(!Sys.isfreebsd, platforms) # freebsd builds fail - fixed in v1.8.0 -filter!(!Sys.iswindows, platforms) # Windows builds fail to link: undefined reference to `faiss::OnDiskInvertedListsIOHook::OnDiskInvertedListsIOHook()' mkl_platforms = Platform[ Platform("x86_64", "Linux"), From 147302db90f06530b952f02e8249facbf8003e55 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Mon, 15 Apr 2024 17:29:01 +0200 Subject: [PATCH 3/9] WIP Added mingw32 patch To make posix_memalign etc. available on mingw32. --- F/Faiss/build_tarballs.jl | 5 ++- F/Faiss/bundled/patches/faiss-mingw32.patch | 44 +++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 F/Faiss/bundled/patches/faiss-mingw32.patch diff --git a/F/Faiss/build_tarballs.jl b/F/Faiss/build_tarballs.jl index f4ff24061e4..143bff0592a 100644 --- a/F/Faiss/build_tarballs.jl +++ b/F/Faiss/build_tarballs.jl @@ -7,7 +7,8 @@ version = v"1.8.0" # Collection of sources required to complete build sources = [ - GitSource("https://github.com/facebookresearch/faiss.git", "22f2292be4aa98559797e1dd6e45f12386b06099") + GitSource("https://github.com/facebookresearch/faiss.git", "22f2292be4aa98559797e1dd6e45f12386b06099"), + DirectorySource("./bundled"), ] # Bash recipe for building across all platforms @@ -17,6 +18,8 @@ apk del cmake cd faiss +atomic_patch -p1 ../patches/faiss-mingw32.patch + cmake -B build \ -DCMAKE_INSTALL_PREFIX=$prefix \ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \ diff --git a/F/Faiss/bundled/patches/faiss-mingw32.patch b/F/Faiss/bundled/patches/faiss-mingw32.patch new file mode 100644 index 00000000000..1d825e0d7ce --- /dev/null +++ b/F/Faiss/bundled/patches/faiss-mingw32.patch @@ -0,0 +1,44 @@ +diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h +index 2aecc512..0c792190 100644 +--- a/faiss/impl/platform_macros.h ++++ b/faiss/impl/platform_macros.h +@@ -11,7 +11,7 @@ + #include + #include + +-#ifdef _MSC_VER ++#ifdef _WIN32 + + /******************************************************* + * Windows specific macros +@@ -23,11 +23,11 @@ + #define FAISS_API __declspec(dllimport) + #endif // FAISS_MAIN_LIB + +-#ifdef _MSC_VER + #define strtok_r strtok_s +-#endif // _MSC_VER + ++#ifdef _MSC_VER + #define __PRETTY_FUNCTION__ __FUNCSIG__ ++#endif // _MSC_VER + + #define posix_memalign(p, a, s) \ + (((*(p)) = _aligned_malloc((s), (a))), *(p) ? 0 : errno) +@@ -37,6 +37,7 @@ + #define ALIGNED(x) __declspec(align(x)) + + // redefine the GCC intrinsics with Windows equivalents ++#ifdef _MSC_VER + + #include + +@@ -91,6 +92,8 @@ inline int __builtin_clzll(uint64_t x) { + #define __F16C__ 1 + #endif + ++#endif // _MSC_VER ++ + #define FAISS_ALWAYS_INLINE __forceinline + + #else From 5b263509a5ca5a9d8dfeb711bbe5715ab39cd0ac Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Wed, 17 Apr 2024 18:06:05 +0200 Subject: [PATCH 4/9] WIP Added Faiss_GPU * Excluded Jetson CUDA archs * Dropped CUDA archs 60, 61 for CUDA 11.8+ --- F/Faiss/Faiss/build_tarballs.jl | 10 +++++++ F/Faiss/Faiss_GPU/build_tarballs.jl | 35 ++++++++++++++++++++++++ F/Faiss/{build_tarballs.jl => common.jl} | 23 +++++++++------- 3 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 F/Faiss/Faiss/build_tarballs.jl create mode 100644 F/Faiss/Faiss_GPU/build_tarballs.jl rename F/Faiss/{build_tarballs.jl => common.jl} (84%) diff --git a/F/Faiss/Faiss/build_tarballs.jl b/F/Faiss/Faiss/build_tarballs.jl new file mode 100644 index 00000000000..c97b12f289e --- /dev/null +++ b/F/Faiss/Faiss/build_tarballs.jl @@ -0,0 +1,10 @@ +# Note that this script can accept some limited command-line arguments, run +# `julia build_tarballs.jl --help` to see a usage message. +using BinaryBuilder, Pkg + +name = "Faiss" + +include(joinpath(@__DIR__, "..", "common.jl")) + +# Build the tarballs, and possibly a `build.jl` as well. +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"7") diff --git a/F/Faiss/Faiss_GPU/build_tarballs.jl b/F/Faiss/Faiss_GPU/build_tarballs.jl new file mode 100644 index 00000000000..bdb927dd2fd --- /dev/null +++ b/F/Faiss/Faiss_GPU/build_tarballs.jl @@ -0,0 +1,35 @@ +# Note that this script can accept some limited command-line arguments, run +# `julia build_tarballs.jl --help` to see a usage message. +using BinaryBuilder, Pkg + +const YGGDRASIL_DIR = "../../.." +include(joinpath(YGGDRASIL_DIR, "fancy_toys.jl")) +include(joinpath(YGGDRASIL_DIR, "platforms", "cuda.jl")) + +name = "Faiss_GPU" + +cuda_versions = ["11.8", "12.1"] # 11.8 to have SM 87;89;90 support. Conda GPU packages build on 11.4 and 12.1: https://github.com/facebookresearch/faiss/blob/v1.8.0/.circleci/config.yml#L358 + +include(joinpath(@__DIR__, "..", "common.jl")) + +# Override the default platforms +platforms = CUDA.supported_platforms() +filter!(p -> arch(p) == "x86_64", platforms) + +filter!(p -> p["cuda"] in cuda_versions, platforms) + +# Build for all supported CUDA toolkits +for platform in platforms + should_build_platform(triplet(platform)) || continue + + cuda_deps = CUDA.required_dependencies(platform) + + build_tarballs(ARGS, name, version, sources, script, [platform], products, [dependencies; cuda_deps]; + lazy_artifacts=true, + julia_compat="1.6", + preferred_gcc_version=v"7", + augment_platform_block=CUDA.augment, + skip_audit=true, + dont_dlopen=true) +end +# trigger yggy diff --git a/F/Faiss/build_tarballs.jl b/F/Faiss/common.jl similarity index 84% rename from F/Faiss/build_tarballs.jl rename to F/Faiss/common.jl index 143bff0592a..96876b8ded8 100644 --- a/F/Faiss/build_tarballs.jl +++ b/F/Faiss/common.jl @@ -1,14 +1,9 @@ -# Note that this script can accept some limited command-line arguments, run -# `julia build_tarballs.jl --help` to see a usage message. -using BinaryBuilder, Pkg - -name = "Faiss" version = v"1.8.0" # Collection of sources required to complete build sources = [ GitSource("https://github.com/facebookresearch/faiss.git", "22f2292be4aa98559797e1dd6e45f12386b06099"), - DirectorySource("./bundled"), + DirectorySource(joinpath(@__DIR__, "bundled")), ] # Bash recipe for building across all platforms @@ -20,6 +15,16 @@ cd faiss atomic_patch -p1 ../patches/faiss-mingw32.patch +cmake_extra_args=() + +if [[ $bb_full_target == *cuda* ]]; then + cmake_extra_args+=( + -DFAISS_ENABLE_GPU=ON + -DCUDAToolkit_ROOT=$prefix/cuda + -DCMAKE_CUDA_ARCHITECTURES="70;75;80;86;89;90" + ) +fi + cmake -B build \ -DCMAKE_INSTALL_PREFIX=$prefix \ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \ @@ -28,7 +33,8 @@ cmake -B build \ -DFAISS_ENABLE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_SHARED_LIBS=ON \ - -DFAISS_ENABLE_C_API=ON + -DFAISS_ENABLE_C_API=ON \ + ${cmake_extra_args[@]} cmake --build build --parallel ${nproc} cmake --install build @@ -70,6 +76,3 @@ dependencies = [ Dependency("OpenBLAS32_jll"; platforms = openblas_platforms), HostBuildDependency(PackageSpec("CMake_jll", v"3.23.3")), ] - -# Build the tarballs, and possibly a `build.jl` as well. -build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"7") From d1363a4f3824a42ed34137863385fc67a6594043 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Thu, 18 Apr 2024 05:57:33 +0200 Subject: [PATCH 5/9] CUDA archs... --- F/Faiss/common.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index 96876b8ded8..ade09089c7f 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -21,7 +21,7 @@ if [[ $bb_full_target == *cuda* ]]; then cmake_extra_args+=( -DFAISS_ENABLE_GPU=ON -DCUDAToolkit_ROOT=$prefix/cuda - -DCMAKE_CUDA_ARCHITECTURES="70;75;80;86;89;90" + -DCMAKE_CUDA_ARCHITECTURES="60" ) fi From 5bf01057723bb302ed81e34691c90f0ca5b7f4f5 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Thu, 18 Apr 2024 06:13:04 +0200 Subject: [PATCH 6/9] Update CMake wrt. CUDA arch detection: https://github.com/Kitware/CMake/blob/v3.23.3/Modules/CMakeDetermineCUDACompiler.cmake#L654 --- F/Faiss/common.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index ade09089c7f..9e104505117 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -74,5 +74,5 @@ dependencies = [ Dependency("MKL_jll"; platforms = mkl_platforms), BuildDependency("MKL_Headers_jll"; platforms = mkl_platforms), Dependency("OpenBLAS32_jll"; platforms = openblas_platforms), - HostBuildDependency(PackageSpec("CMake_jll", v"3.23.3")), + HostBuildDependency(PackageSpec("CMake_jll", v"3.28.1")), ] From 9498a4dd62ce78fabde6a5b7ee1dcdd923e69ba3 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Thu, 18 Apr 2024 06:18:55 +0200 Subject: [PATCH 7/9] set CUDACXX --- F/Faiss/common.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index 9e104505117..a6efb6efe1f 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -18,6 +18,7 @@ atomic_patch -p1 ../patches/faiss-mingw32.patch cmake_extra_args=() if [[ $bb_full_target == *cuda* ]]; then + export CUDACXX=$prefix/cuda/bin/nvcc cmake_extra_args+=( -DFAISS_ENABLE_GPU=ON -DCUDAToolkit_ROOT=$prefix/cuda From 1f911be558aee490394c4f19a5e1f401fd147038 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Thu, 18 Apr 2024 06:28:04 +0200 Subject: [PATCH 8/9] cudart --- F/Faiss/common.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index a6efb6efe1f..3a0e7911ec5 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -23,6 +23,7 @@ if [[ $bb_full_target == *cuda* ]]; then -DFAISS_ENABLE_GPU=ON -DCUDAToolkit_ROOT=$prefix/cuda -DCMAKE_CUDA_ARCHITECTURES="60" + -DCUDA_CUDART_LIBRARY=$prefix/cuda/lib64/libcudart.$dlext ) fi From 7e7b18aa7a9bdd984a7dcc4eb105bed8ae2dc468 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Thu, 18 Apr 2024 06:28:18 +0200 Subject: [PATCH 9/9] Include CUDA static SDK --- F/Faiss/Faiss_GPU/build_tarballs.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/F/Faiss/Faiss_GPU/build_tarballs.jl b/F/Faiss/Faiss_GPU/build_tarballs.jl index bdb927dd2fd..06298710fdc 100644 --- a/F/Faiss/Faiss_GPU/build_tarballs.jl +++ b/F/Faiss/Faiss_GPU/build_tarballs.jl @@ -22,7 +22,7 @@ filter!(p -> p["cuda"] in cuda_versions, platforms) for platform in platforms should_build_platform(triplet(platform)) || continue - cuda_deps = CUDA.required_dependencies(platform) + cuda_deps = CUDA.required_dependencies(platform; static_sdk=true) build_tarballs(ARGS, name, version, sources, script, [platform], products, [dependencies; cuda_deps]; lazy_artifacts=true,