From e04f8e1c7f8fa20d1ba75669e44e5671acd9d759 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Mon, 27 Jan 2025 09:46:40 +0100 Subject: [PATCH 1/7] [Faiss] Replaced MKL with libblastrampoline Due to issue with linking with MKL: https://github.com/JuliaPackaging/Yggdrasil/pull/8572#issuecomment-2614600730 Also, removes the need to build with a newer GCC (than v7) wrt. OpenBLAS on PowerPC64LE. --- F/Faiss/Faiss/build_tarballs.jl | 4 +- F/Faiss/Faiss_CUDA/build_tarballs.jl | 2 +- .../patches/faiss-cmake-mkl-optional.patch | 27 +++++++++++++ F/Faiss/common.jl | 39 ++++++++++++------- 4 files changed, 54 insertions(+), 18 deletions(-) create mode 100644 F/Faiss/bundled/patches/faiss-cmake-mkl-optional.patch diff --git a/F/Faiss/Faiss/build_tarballs.jl b/F/Faiss/Faiss/build_tarballs.jl index bca8fa27748..0b7d7956741 100644 --- a/F/Faiss/Faiss/build_tarballs.jl +++ b/F/Faiss/Faiss/build_tarballs.jl @@ -9,6 +9,6 @@ include(joinpath(@__DIR__, "..", "common.jl")) # Build the tarballs, and possibly a `build.jl` as well. build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; - julia_compat="1.6", - preferred_gcc_version=v"10", # Not using v"7", since OpenBLAS v0.3.29+ on PowerPC64LE requires libgfortran5, and not using v"8", and v"9" due to internal compiler errors on aarch64-linux-gnu + julia_compat="1.9", + preferred_gcc_version=v"7", ) diff --git a/F/Faiss/Faiss_CUDA/build_tarballs.jl b/F/Faiss/Faiss_CUDA/build_tarballs.jl index d0fc49f2e45..607f5a66911 100644 --- a/F/Faiss/Faiss_CUDA/build_tarballs.jl +++ b/F/Faiss/Faiss_CUDA/build_tarballs.jl @@ -39,7 +39,7 @@ for platform in platforms build_tarballs(ARGS, name, version, sources, script, [platform], products, [dependencies; cuda_deps]; lazy_artifacts=true, - julia_compat="1.6", + julia_compat="1.9", preferred_gcc_version=v"7", augment_platform_block=CUDA.augment, skip_audit=true, diff --git a/F/Faiss/bundled/patches/faiss-cmake-mkl-optional.patch b/F/Faiss/bundled/patches/faiss-cmake-mkl-optional.patch new file mode 100644 index 00000000000..a61215b778e --- /dev/null +++ b/F/Faiss/bundled/patches/faiss-cmake-mkl-optional.patch @@ -0,0 +1,27 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index c4fea46a..12f8aef5 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -59,6 +59,7 @@ list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") + # Valid values are "generic", "avx2", "avx512", "sve". + option(FAISS_OPT_LEVEL "" "generic") + option(FAISS_ENABLE_GPU "Enable support for GPU indexes." ON) ++option(FAISS_ENABLE_MKL "Enable MKL." ON) + option(FAISS_ENABLE_RAFT "Enable RAFT for GPU indexes." OFF) + option(FAISS_ENABLE_ROCM "Enable ROCm for GPU indexes." OFF) + option(FAISS_ENABLE_PYTHON "Build Python extension." ON) +diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt +index 2871d974..6588b2d2 100644 +--- a/faiss/CMakeLists.txt ++++ b/faiss/CMakeLists.txt +@@ -354,7 +354,9 @@ target_link_libraries(faiss_avx2 PRIVATE OpenMP::OpenMP_CXX) + target_link_libraries(faiss_avx512 PRIVATE OpenMP::OpenMP_CXX) + target_link_libraries(faiss_sve PRIVATE OpenMP::OpenMP_CXX) + +-find_package(MKL) ++if(FAISS_ENABLE_MKL) ++ find_package(MKL) ++endif() + if(MKL_FOUND) + target_link_libraries(faiss PRIVATE ${MKL_LIBRARIES}) + target_link_libraries(faiss_avx2 PRIVATE ${MKL_LIBRARIES}) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index d31a935b233..44ad37d4b99 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -11,12 +11,26 @@ apk del cmake cd faiss +atomic_patch -p1 ../patches/faiss-cmake-mkl-optional.patch atomic_patch -p1 ../patches/faiss-mingw32-cmake.patch atomic_patch -p1 ../patches/faiss-mingw32-InvertedListsIOHook.patch atomic_patch -p1 ../patches/faiss-mingw32.patch cmake_extra_args=() +libblastrampoline_target=$(echo $bb_full_target | cut -d- -f 1-3) +if [[ "$libblastrampoline_target" != armv6l-linux-* && + "$bb_full_target" != i686-linux-gnu-cxx11 ]]; then + if [[ "$target" == *-freebsd* ]]; then + libblastrampoline_target=$rust_target + fi + cmake_extra_args+=( + "-DBLAS_INCLUDE_DIRS=$includedir/libblastrampoline/LP64/$libblastrampoline_target" + "-DBLAS_LIBRARIES=$libdir/libblastrampoline.$dlext" + "-DLAPACK_INCLUDE_DIRS=$includedir/libblastrampoline/LP64/$libblastrampoline_target" + ) +fi + if [[ $bb_full_target == *cuda* ]]; then cuda_version=${bb_full_target##*-cuda+} if [[ $cuda_version == "11.8" ]]; then @@ -45,6 +59,7 @@ cmake -B build \ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \ -DCMAKE_BUILD_TYPE=Release \ -DFAISS_ENABLE_GPU=OFF \ + -DFAISS_ENABLE_MKL=OFF \ -DFAISS_ENABLE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_SHARED_LIBS=ON \ @@ -62,19 +77,15 @@ fi # These are the platforms we will build for by default, unless further # platforms are passed in on the command line -platforms = supported_platforms() - -mkl_platforms = Platform[ - Platform("x86_64", "Linux"), - Platform("x86_64", "MacOS"), - Platform("x86_64", "Windows"), -] - -openblas_platforms = filter(p -> p ∉ mkl_platforms, platforms) +platforms = expand_cxxstring_abis(supported_platforms()) -platforms = expand_cxxstring_abis(platforms) -mkl_platforms = expand_cxxstring_abis(mkl_platforms) -openblas_platforms = expand_cxxstring_abis(openblas_platforms) +openblas_platforms = filter(p -> + arch(p) == "armv6l" || + p == Platform("i686", "Linux"; libc = "glibc", cxxstring_abi = "cxx11") || + Sys.isfreebsd(p) && arch(p) == "aarch64", + platforms +) +libblastrampoline_platforms = filter(p -> p ∉ openblas_platforms, platforms) # The products that we will ensure are always built products = Product[ @@ -90,9 +101,7 @@ dependencies = [ # systems), and libgomp from `CompilerSupportLibraries_jll` everywhere else. Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"); platforms=filter(!Sys.isbsd, platforms)), Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e"); platforms=filter(Sys.isbsd, platforms)), - Dependency("LAPACK_jll"; platforms = openblas_platforms), - Dependency("MKL_jll"; platforms = mkl_platforms), - BuildDependency("MKL_Headers_jll"; platforms = mkl_platforms), + Dependency("libblastrampoline_jll"; compat="5.4", platforms = libblastrampoline_platforms), Dependency("OpenBLAS32_jll"; platforms = openblas_platforms), HostBuildDependency(PackageSpec("CMake_jll", v"3.28.1")), ] From 719418e460aa1ef3917ca1f58e522153d1f730c3 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Mon, 27 Jan 2025 10:28:23 +0100 Subject: [PATCH 2/7] Reduced lbt configure --- F/Faiss/common.jl | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index 44ad37d4b99..5bfcc20abea 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -18,16 +18,10 @@ atomic_patch -p1 ../patches/faiss-mingw32.patch cmake_extra_args=() -libblastrampoline_target=$(echo $bb_full_target | cut -d- -f 1-3) -if [[ "$libblastrampoline_target" != armv6l-linux-* && +if [[ "$bb_full_target" != armv6l-linux-* && "$bb_full_target" != i686-linux-gnu-cxx11 ]]; then - if [[ "$target" == *-freebsd* ]]; then - libblastrampoline_target=$rust_target - fi cmake_extra_args+=( - "-DBLAS_INCLUDE_DIRS=$includedir/libblastrampoline/LP64/$libblastrampoline_target" "-DBLAS_LIBRARIES=$libdir/libblastrampoline.$dlext" - "-DLAPACK_INCLUDE_DIRS=$includedir/libblastrampoline/LP64/$libblastrampoline_target" ) fi From 6b32f4ce250dd191c22546e818562cdedab79000 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Mon, 27 Jan 2025 10:44:45 +0100 Subject: [PATCH 3/7] Reduced lbt configure --- F/Faiss/common.jl | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index 5bfcc20abea..c8fc31eb8e4 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -18,10 +18,10 @@ atomic_patch -p1 ../patches/faiss-mingw32.patch cmake_extra_args=() -if [[ "$bb_full_target" != armv6l-linux-* && - "$bb_full_target" != i686-linux-gnu-cxx11 ]]; then +if [[ "$bb_full_target" != armv6l-linux-* ]]; then cmake_extra_args+=( "-DBLAS_LIBRARIES=$libdir/libblastrampoline.$dlext" + "-DLAPACK_LIBRARIES=$libdir/libblastrampoline.$dlext" ) fi @@ -73,12 +73,7 @@ fi # platforms are passed in on the command line platforms = expand_cxxstring_abis(supported_platforms()) -openblas_platforms = filter(p -> - arch(p) == "armv6l" || - p == Platform("i686", "Linux"; libc = "glibc", cxxstring_abi = "cxx11") || - Sys.isfreebsd(p) && arch(p) == "aarch64", - platforms -) +openblas_platforms = filter(p -> arch(p) == "armv6l", platforms) libblastrampoline_platforms = filter(p -> p ∉ openblas_platforms, platforms) # The products that we will ensure are always built From 99167b4843b5fd5e51d68ec8fb3773df6a21757d Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Tue, 28 Jan 2025 11:18:38 +0100 Subject: [PATCH 4/7] Enabled Windows builds to find lbt --- F/Faiss/common.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index c8fc31eb8e4..9e1874275ea 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -19,9 +19,10 @@ atomic_patch -p1 ../patches/faiss-mingw32.patch cmake_extra_args=() if [[ "$bb_full_target" != armv6l-linux-* ]]; then + libblastrampoline=$(find $libdir -name libblastrampoline'*'.$dlext) # Enable Windows builds to find libblastrampoline cmake_extra_args+=( - "-DBLAS_LIBRARIES=$libdir/libblastrampoline.$dlext" - "-DLAPACK_LIBRARIES=$libdir/libblastrampoline.$dlext" + "-DBLAS_LIBRARIES=$libblastrampoline" + "-DLAPACK_LIBRARIES=$libblastrampoline" ) fi From 54bfc2f83b4896e6cc39745e580643e247959ade Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Tue, 28 Jan 2025 11:37:21 +0100 Subject: [PATCH 5/7] Enabled lbt for armv6l --- F/Faiss/common.jl | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index 9e1874275ea..24581b10f09 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -18,14 +18,6 @@ atomic_patch -p1 ../patches/faiss-mingw32.patch cmake_extra_args=() -if [[ "$bb_full_target" != armv6l-linux-* ]]; then - libblastrampoline=$(find $libdir -name libblastrampoline'*'.$dlext) # Enable Windows builds to find libblastrampoline - cmake_extra_args+=( - "-DBLAS_LIBRARIES=$libblastrampoline" - "-DLAPACK_LIBRARIES=$libblastrampoline" - ) -fi - if [[ $bb_full_target == *cuda* ]]; then cuda_version=${bb_full_target##*-cuda+} if [[ $cuda_version == "11.8" ]]; then @@ -49,16 +41,21 @@ if [[ $bb_full_target == *cuda* ]]; then ) fi +libblastrampoline=$(find $libdir -name libblastrampoline'*'.$dlext) # Enable Windows builds to find libblastrampoline +cmake_extra_args+=( + "-DBLAS_LIBRARIES=$libblastrampoline" + "-DLAPACK_LIBRARIES=$libblastrampoline" +) cmake -B build \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=ON \ -DCMAKE_INSTALL_PREFIX=$prefix \ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \ -DCMAKE_BUILD_TYPE=Release \ + -DFAISS_ENABLE_C_API=ON \ -DFAISS_ENABLE_GPU=OFF \ -DFAISS_ENABLE_MKL=OFF \ -DFAISS_ENABLE_PYTHON=OFF \ - -DBUILD_TESTING=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DFAISS_ENABLE_C_API=ON \ ${cmake_extra_args[@]} cmake --build build --parallel ${nproc} cmake --install build @@ -74,9 +71,6 @@ fi # platforms are passed in on the command line platforms = expand_cxxstring_abis(supported_platforms()) -openblas_platforms = filter(p -> arch(p) == "armv6l", platforms) -libblastrampoline_platforms = filter(p -> p ∉ openblas_platforms, platforms) - # The products that we will ensure are always built products = Product[ FileProduct("include/faiss/Index.h", :faiss_index_h), @@ -91,7 +85,6 @@ dependencies = [ # systems), and libgomp from `CompilerSupportLibraries_jll` everywhere else. Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"); platforms=filter(!Sys.isbsd, platforms)), Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e"); platforms=filter(Sys.isbsd, platforms)), - Dependency("libblastrampoline_jll"; compat="5.4", platforms = libblastrampoline_platforms), - Dependency("OpenBLAS32_jll"; platforms = openblas_platforms), + Dependency("libblastrampoline_jll"; compat="5.4"), HostBuildDependency(PackageSpec("CMake_jll", v"3.28.1")), ] From 3485be9ab045ad8ef81efdd00cc13f214e3fd8ae Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Tue, 28 Jan 2025 12:09:21 +0100 Subject: [PATCH 6/7] Configured lbt via CMake 3.30 BLA_VENDOR variable --- F/Faiss/common.jl | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl index 24581b10f09..e6ae9f86c83 100644 --- a/F/Faiss/common.jl +++ b/F/Faiss/common.jl @@ -41,17 +41,13 @@ if [[ $bb_full_target == *cuda* ]]; then ) fi -libblastrampoline=$(find $libdir -name libblastrampoline'*'.$dlext) # Enable Windows builds to find libblastrampoline -cmake_extra_args+=( - "-DBLAS_LIBRARIES=$libblastrampoline" - "-DLAPACK_LIBRARIES=$libblastrampoline" -) cmake -B build \ - -DBUILD_TESTING=OFF \ + -DBLA_VENDOR=libblastrampoline \ -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=$prefix \ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \ - -DCMAKE_BUILD_TYPE=Release \ -DFAISS_ENABLE_C_API=ON \ -DFAISS_ENABLE_GPU=OFF \ -DFAISS_ENABLE_MKL=OFF \ @@ -86,5 +82,5 @@ dependencies = [ Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"); platforms=filter(!Sys.isbsd, platforms)), Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e"); platforms=filter(Sys.isbsd, platforms)), Dependency("libblastrampoline_jll"; compat="5.4"), - HostBuildDependency(PackageSpec("CMake_jll", v"3.28.1")), + HostBuildDependency(PackageSpec("CMake_jll", v"3.30.2")), ] From 84355b8e22cb49cdfc597ca71988f516dd728b04 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Thu, 30 Jan 2025 10:01:19 +0100 Subject: [PATCH 7/7] Removed yggy comment --- F/Faiss/Faiss_CUDA/build_tarballs.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/F/Faiss/Faiss_CUDA/build_tarballs.jl b/F/Faiss/Faiss_CUDA/build_tarballs.jl index 607f5a66911..f5c523d3fb7 100644 --- a/F/Faiss/Faiss_CUDA/build_tarballs.jl +++ b/F/Faiss/Faiss_CUDA/build_tarballs.jl @@ -45,4 +45,3 @@ for platform in platforms skip_audit=true, dont_dlopen=true) end -# trigger yggy