Skip to content

Commit 514b44f

Browse files
Michael Norrisfacebook-github-bot
Michael Norris
authored and
facebook-github-bot
committed
OSS changes: Enable ROCm to work with Faiss on BUCK (#4485)
Summary: Pull Request resolved: #4485 hipify.sh currently generates .hip files into a separate directory called `gpu-rocm`. The swigfaiss.swig file expects them there too. But to use internal hipification, it needs to be generated into the same directory `gpu`. That means swigfaiss.swig has to change `gpu-rocm` to just `gpu`, same directory as .cu files. If swigfaiss.swig changes, then the hipify.sh has to change as well, otherwise OSS CI breaks. OSS changes - Requires changing `swigfaiss.swig`, because the internal BUCK approach builds in the same directory as `.cu` files, aka `gpu/`. - Because it builds in same `gpu/` directory, we need to change the OSS side `hipify.sh`. After this diff, it just builds in `gpu/`. The rocm CI still passes. Reviewed By: junjieqi Differential Revision: D79145713 fbshipit-source-id: a83b805c8a7bf286a7e6cff50c8eb2ddd72f518c
1 parent dd637c9 commit 514b44f

5 files changed

Lines changed: 48 additions & 199 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -91,11 +91,7 @@ if(FAISS_ENABLE_CUVS AND NOT TARGET cuvs::cuvs)
9191
add_subdirectory(faiss)
9292

9393
if(FAISS_ENABLE_GPU)
94-
if(FAISS_ENABLE_ROCM)
95-
add_subdirectory(faiss/gpu-rocm)
96-
else()
97-
add_subdirectory(faiss/gpu)
98-
endif()
94+
add_subdirectory(faiss/gpu)
9995
endif()
10096

10197
if(FAISS_ENABLE_PYTHON)
@@ -119,7 +115,7 @@ if(BUILD_TESTING)
119115
add_subdirectory(perf_tests)
120116
if(FAISS_ENABLE_GPU)
121117
if(FAISS_ENABLE_ROCM)
122-
add_subdirectory(faiss/gpu-rocm/test)
118+
add_subdirectory(faiss/gpu/test)
123119
else()
124120
add_subdirectory(faiss/gpu/test)
125121
endif()

‎c_api/CMakeLists.txt‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -160,9 +160,5 @@ add_executable(example_c EXCLUDE_FROM_ALL example_c.c)
160160
target_link_libraries(example_c PRIVATE faiss_c)
161161

162162
if(FAISS_ENABLE_GPU)
163-
if(FAISS_ENABLE_ROCM)
164-
add_subdirectory(gpu-rocm)
165-
else ()
166-
add_subdirectory(gpu)
167-
endif()
163+
add_subdirectory(gpu)
168164
endif()

‎faiss/gpu/hipify.sh‎

Lines changed: 13 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -39,29 +39,22 @@ function hipify_dir()
3939
exit
4040
fi
4141

42-
# create all destination directories for hipified files into sibling 'gpu-rocm' directory
43-
while IFS= read -r -d '' src
44-
do
45-
dst="${src//gpu-tmp/gpu-rocm}"
46-
47-
if [ -d $dst ]; then
48-
#Clearing out any leftover files and directories
49-
echo "Removing old $dst"
50-
rm -rf "$dst"
51-
fi
52-
53-
#Making directories
54-
echo "Creating $dst"
55-
mkdir -p "$dst"
56-
done < <(find ./gpu-tmp -type d -print0)
42+
# Create a backup of the original gpu directory
43+
if [ -d ./gpu-backup ]; then
44+
echo "Removing old ./gpu-backup"
45+
rm -rf ./gpu-backup
46+
fi
47+
echo "Creating backup of original gpu directory"
48+
cp -r ./gpu ./gpu-backup
5749

5850
# run hipify-perl against all *.cu *.cuh *.h *.cpp files, no renaming
5951
# run all files in parallel to speed up
6052
for ext in cu cuh h cpp c
6153
do
6254
while IFS= read -r -d '' src
6355
do
64-
dst="${src//\.\/gpu-tmp/\.\/gpu-rocm}"
56+
# Output to the original gpu directory but with .tmp extension
57+
dst="${src//\.\/gpu-tmp/\.\/gpu}"
6558
hipify-perl -o="$dst.tmp" "$src" &
6659
done < <(find ./gpu-tmp -name "*.$ext" -print0)
6760
done
@@ -72,27 +65,25 @@ function hipify_dir()
7265
do
7366
dst=${src%.cu.tmp}.hip.tmp
7467
mv "$src" "$dst"
75-
done < <(find ./gpu-rocm -name "*.cu.tmp" -print0)
68+
done < <(find ./gpu -name "*.cu.tmp" -print0)
7669

7770
if [ -d ./gpu-tmp ]; then
7871
#Clearing out any leftover files and directories
7972
echo "Removing ./gpu-tmp"
8073
rm -rf ./gpu-tmp
8174
fi
8275

83-
# replace header include statements "<faiss/gpu/" with "<faiss/gpu-rocm"
8476
# replace thrust::cuda::par with thrust::hip::par
8577
# adjust header path location for hipblas.h to avoid unnecessary deprecation warnings
8678
# adjust header path location for hiprand_kernel.h to avoid unnecessary deprecation warnings
8779
for ext in hip cuh h cpp c
8880
do
8981
while IFS= read -r -d '' src
9082
do
91-
sed -i 's@#include <faiss/gpu/@#include <faiss/gpu-rocm/@' "$src"
9283
sed -i 's@thrust::cuda::par@thrust::hip::par@' "$src"
9384
sed -i 's@#include <hipblas.h>@#include <hipblas/hipblas.h>@' "$src"
9485
sed -i 's@#include <hiprand_kernel.h>@#include <hiprand/hiprand_kernel.h>@' "$src"
95-
done < <(find ./gpu-rocm -name "*.$ext.tmp" -print0)
86+
done < <(find ./gpu -name "*.$ext.tmp" -print0)
9687
done
9788

9889
# hipify was run in parallel above
@@ -116,50 +107,10 @@ function hipify_dir()
116107
echo "$dst"
117108
mv "$src" "$dst"
118109
fi
119-
done < <(find ./gpu-rocm -name "*.$ext.tmp" -print0)
110+
done < <(find ./gpu -name "*.$ext.tmp" -print0)
120111
done
121112

122-
# copy over CMakeLists.txt
123-
while IFS= read -r -d '' src
124-
do
125-
dst="${src//\.\/gpu/\.\/gpu-rocm}"
126-
if test -f "$dst"
127-
then
128-
if diff -q "$src" "$dst" >& /dev/null
129-
then
130-
echo "$dst [unchanged]"
131-
else
132-
echo "$dst"
133-
cp "$src" "$dst"
134-
fi
135-
else
136-
echo "$dst"
137-
cp "$src" "$dst"
138-
fi
139-
done < <(find ./gpu -name "CMakeLists.txt" -print0)
140-
141-
# Copy over other files
142-
other_exts="py"
143-
for ext in $other_exts
144-
do
145-
while IFS= read -r -d '' src
146-
do
147-
dst="${src//\.\/gpu/\.\/gpu-rocm}"
148-
if test -f "$dst"
149-
then
150-
if diff -q "$src" "$dst" >& /dev/null
151-
then
152-
echo "$dst [unchanged]"
153-
else
154-
echo "$dst"
155-
cp "$src" "$dst"
156-
fi
157-
else
158-
echo "$dst"
159-
cp "$src" "$dst"
160-
fi
161-
done < <(find ./gpu -name "*.$ext" -print0)
162-
done
113+
# No need to copy CMakeLists.txt or other files as we're working directly in the gpu directory
163114
}
164115

165116
# Convert the faiss/gpu dir

‎faiss/python/CMakeLists.txt‎

Lines changed: 14 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -81,35 +81,20 @@ if(TARGET faiss)
8181
list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
8282
"${faiss_SOURCE_DIR}/faiss/${h}")
8383
endforeach()
84-
if(FAISS_ENABLE_ROCM)
85-
foreach(h ${FAISS_GPU_HEADERS})
86-
list(APPEND SWIG_MODULE_swigfaiss_EXTRA_DEPS
87-
"${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
88-
list(APPEND SWIG_MODULE_swigfaiss_avx2_EXTRA_DEPS
89-
"${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
90-
list(APPEND SWIG_MODULE_swigfaiss_avx512_EXTRA_DEPS
91-
"${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
92-
list(APPEND SWIG_MODULE_swigfaiss_avx512_spr_EXTRA_DEPS
93-
"${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
94-
list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
95-
"${faiss_SOURCE_DIR}/faiss/gpu-rocm/${h}")
96-
endforeach()
97-
else()
98-
foreach(h ${FAISS_GPU_HEADERS})
99-
list(APPEND SWIG_MODULE_swigfaiss_EXTRA_DEPS
100-
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
101-
list(APPEND SWIG_MODULE_swigfaiss_avx2_EXTRA_DEPS
102-
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
103-
list(APPEND SWIG_MODULE_swigfaiss_avx512_EXTRA_DEPS
104-
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
105-
list(APPEND SWIG_MODULE_swigfaiss_avx512_spr_EXTRA_DEPS
106-
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
107-
list(APPEND SWIG_MODULE_swigfaiss_sve_EXTRA_DEPS
108-
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
109-
list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
110-
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
111-
endforeach()
112-
endif()
84+
foreach(h ${FAISS_GPU_HEADERS})
85+
list(APPEND SWIG_MODULE_swigfaiss_EXTRA_DEPS
86+
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
87+
list(APPEND SWIG_MODULE_swigfaiss_avx2_EXTRA_DEPS
88+
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
89+
list(APPEND SWIG_MODULE_swigfaiss_avx512_EXTRA_DEPS
90+
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
91+
list(APPEND SWIG_MODULE_swigfaiss_avx512_spr_EXTRA_DEPS
92+
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
93+
list(APPEND SWIG_MODULE_swigfaiss_sve_EXTRA_DEPS
94+
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
95+
list(APPEND SWIG_MODULE_faiss_example_external_module_EXTRA_DEPS
96+
"${faiss_SOURCE_DIR}/faiss/gpu/${h}")
97+
endforeach()
11398
else()
11499
find_package(faiss REQUIRED)
115100
endif()

‎faiss/python/swigfaiss.swig‎

Lines changed: 18 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -330,27 +330,26 @@ void gpu_profiler_stop();
330330
void gpu_sync_all_devices();
331331

332332
#ifdef GPU_WRAPPER
333-
#ifdef FAISS_ENABLE_ROCM
334333

335334
%shared_ptr(faiss::gpu::GpuResources);
336335
%shared_ptr(faiss::gpu::StandardGpuResourcesImpl);
337336

338337
%{
339338

340-
#include <faiss/gpu-rocm/StandardGpuResources.h>
341-
#include <faiss/gpu-rocm/GpuIndicesOptions.h>
342-
#include <faiss/gpu-rocm/GpuClonerOptions.h>
343-
#include <faiss/gpu-rocm/GpuIndex.h>
344-
#include <faiss/gpu-rocm/GpuIndexFlat.h>
345-
#include <faiss/gpu-rocm/GpuIndexIVF.h>
346-
#include <faiss/gpu-rocm/GpuIndexIVFPQ.h>
347-
#include <faiss/gpu-rocm/GpuIndexIVFFlat.h>
348-
#include <faiss/gpu-rocm/GpuIndexIVFScalarQuantizer.h>
349-
#include <faiss/gpu-rocm/GpuIndexBinaryFlat.h>
350-
#include <faiss/gpu-rocm/GpuAutoTune.h>
351-
#include <faiss/gpu-rocm/GpuCloner.h>
352-
#include <faiss/gpu-rocm/GpuDistance.h>
353-
#include <faiss/gpu-rocm/GpuIcmEncoder.h>
339+
#include <faiss/gpu/StandardGpuResources.h>
340+
#include <faiss/gpu/GpuIndicesOptions.h>
341+
#include <faiss/gpu/GpuClonerOptions.h>
342+
#include <faiss/gpu/GpuIndex.h>
343+
#include <faiss/gpu/GpuIndexFlat.h>
344+
#include <faiss/gpu/GpuIndexIVF.h>
345+
#include <faiss/gpu/GpuIndexIVFPQ.h>
346+
#include <faiss/gpu/GpuIndexIVFFlat.h>
347+
#include <faiss/gpu/GpuIndexIVFScalarQuantizer.h>
348+
#include <faiss/gpu/GpuIndexBinaryFlat.h>
349+
#include <faiss/gpu/GpuAutoTune.h>
350+
#include <faiss/gpu/GpuCloner.h>
351+
#include <faiss/gpu/GpuDistance.h>
352+
#include <faiss/gpu/GpuIcmEncoder.h>
354353

355354
int get_num_gpus()
356355
{
@@ -384,8 +383,10 @@ void gpu_sync_all_devices()
384383
%ignore faiss::gpu::GpuMemoryReservation::operator=(GpuMemoryReservation&&);
385384
%ignore faiss::gpu::AllocType;
386385

387-
%include <faiss/gpu-rocm/GpuResources.h>
388-
%include <faiss/gpu-rocm/StandardGpuResources.h>
386+
%include <faiss/gpu/GpuResources.h>
387+
%include <faiss/gpu/StandardGpuResources.h>
388+
389+
#ifdef FAISS_ENABLE_ROCM
389390

390391
typedef ihipStream_t* hipStream_t;
391392

@@ -404,65 +405,16 @@ int64_t cast_cudastream_t_to_integer(hipStream_t x) {
404405

405406
#else // FAISS_ENABLE_ROCM
406407

407-
%shared_ptr(faiss::gpu::GpuResources);
408-
%shared_ptr(faiss::gpu::StandardGpuResourcesImpl);
409408
%shared_ptr(faiss::gpu::IVFPQBuildCagraConfig);
410409
%shared_ptr(faiss::gpu::IVFPQSearchCagraConfig);
411410

412411
%{
413412

414-
#include <faiss/gpu/StandardGpuResources.h>
415-
#include <faiss/gpu/GpuIndicesOptions.h>
416-
#include <faiss/gpu/GpuClonerOptions.h>
417-
#include <faiss/gpu/GpuIndex.h>
418413
#include <faiss/gpu/GpuIndexCagra.h>
419414
#include <faiss/gpu/GpuIndexBinaryCagra.h>
420-
#include <faiss/gpu/GpuIndexFlat.h>
421-
#include <faiss/gpu/GpuIndexIVF.h>
422-
#include <faiss/gpu/GpuIndexIVFPQ.h>
423-
#include <faiss/gpu/GpuIndexIVFFlat.h>
424-
#include <faiss/gpu/GpuIndexIVFScalarQuantizer.h>
425-
#include <faiss/gpu/GpuIndexBinaryFlat.h>
426-
#include <faiss/gpu/GpuAutoTune.h>
427-
#include <faiss/gpu/GpuCloner.h>
428-
#include <faiss/gpu/GpuDistance.h>
429-
#include <faiss/gpu/GpuIcmEncoder.h>
430-
431-
int get_num_gpus()
432-
{
433-
return faiss::gpu::getNumDevices();
434-
}
435-
436-
void gpu_profiler_start()
437-
{
438-
return faiss::gpu::profilerStart();
439-
}
440-
441-
void gpu_profiler_stop()
442-
{
443-
return faiss::gpu::profilerStop();
444-
}
445-
446-
void gpu_sync_all_devices()
447-
{
448-
return faiss::gpu::synchronizeAllDevices();
449-
}
450415

451416
%}
452417

453-
%template() std::pair<int, uint64_t>;
454-
%template() std::map<std::string, std::pair<int, uint64_t> >;
455-
%template() std::map<int, std::map<std::string, std::pair<int, uint64_t> > >;
456-
457-
// causes weird wrapper bug
458-
%ignore *::allocMemoryHandle;
459-
%ignore faiss::gpu::GpuMemoryReservation;
460-
%ignore faiss::gpu::GpuMemoryReservation::operator=(GpuMemoryReservation&&);
461-
%ignore faiss::gpu::AllocType;
462-
463-
%include <faiss/gpu/GpuResources.h>
464-
%include <faiss/gpu/StandardGpuResources.h>
465-
466418
typedef CUstream_st* cudaStream_t;
467419

468420
%inline %{
@@ -682,25 +634,6 @@ struct faiss::simd16uint16 {};
682634

683635
#ifdef GPU_WRAPPER
684636

685-
#ifdef FAISS_ENABLE_ROCM
686-
687-
// quiet SWIG warnings
688-
%ignore faiss::gpu::GpuIndexIVF::GpuIndexIVF;
689-
690-
%include <faiss/gpu-rocm/GpuIndicesOptions.h>
691-
%include <faiss/gpu-rocm/GpuClonerOptions.h>
692-
%include <faiss/gpu-rocm/GpuIndex.h>
693-
%include <faiss/gpu-rocm/GpuIndexFlat.h>
694-
%include <faiss/gpu-rocm/GpuIndexIVF.h>
695-
%include <faiss/gpu-rocm/GpuIndexIVFPQ.h>
696-
%include <faiss/gpu-rocm/GpuIndexIVFFlat.h>
697-
%include <faiss/gpu-rocm/GpuIndexIVFScalarQuantizer.h>
698-
%include <faiss/gpu-rocm/GpuIndexBinaryFlat.h>
699-
%include <faiss/gpu-rocm/GpuDistance.h>
700-
%include <faiss/gpu-rocm/GpuIcmEncoder.h>
701-
702-
#else // FAISS_ENABLE_ROCM
703-
704637
// quiet SWIG warnings
705638
%ignore faiss::gpu::GpuIndexIVF::GpuIndexIVF;
706639

@@ -720,7 +653,6 @@ struct faiss::simd16uint16 {};
720653
%include <faiss/gpu/GpuDistance.h>
721654
%include <faiss/gpu/GpuIcmEncoder.h>
722655

723-
#endif // FAISS_ENABLE_ROCM
724656
#endif
725657

726658

@@ -960,16 +892,6 @@ faiss::Quantizer * downcast_Quantizer (faiss::Quantizer *aq)
960892

961893
#ifdef GPU_WRAPPER
962894

963-
#ifdef FAISS_ENABLE_ROCM
964-
%include <faiss/gpu-rocm/GpuAutoTune.h>
965-
966-
%newobject index_gpu_to_cpu;
967-
%newobject index_cpu_to_gpu;
968-
%newobject index_cpu_to_gpu_multiple;
969-
970-
%include <faiss/gpu-rocm/GpuCloner.h>
971-
972-
#else // FAISS_ENABLE_ROCM
973895
%include <faiss/gpu/GpuAutoTune.h>
974896

975897
%newobject index_gpu_to_cpu;
@@ -978,7 +900,6 @@ faiss::Quantizer * downcast_Quantizer (faiss::Quantizer *aq)
978900

979901
%include <faiss/gpu/GpuCloner.h>
980902

981-
#endif // FAISS_ENABLE_ROCM
982903
#endif
983904

984905

0 commit comments

Comments
 (0)