Skip to content

Commit f9a01c6

Browse files
ItsPittfacebook-github-bot
ItsPitt
authored and
facebook-github-bot
committed
Removing Manual Hipify Build Step (facebookresearch#3962)
Summary: - Called the hipify script at CMAKE configure time removing the need for the user to run it. - Now removes any .hip files left over when running the hipify script. - Cleaned up the hipify script to remove redundancy. Pull Request resolved: facebookresearch#3962 Reviewed By: asadoughi, ramilbakhshyiev Differential Revision: D64495550 Pulled By: mnorris11 fbshipit-source-id: 5547712a4e46fc18cf62346adb0395d0e5626399
1 parent 6617b13 commit f9a01c6

4 files changed

Lines changed: 95 additions & 180 deletions

File tree

‎.github/actions/build_cmake/action.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,6 @@ runs:
101101
sudo apt-get -qq autoclean >/dev/null
102102
sudo apt-get -qq clean >/dev/null
103103
sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
104-
- name: ROCm - Hipify
105-
if: inputs.rocm == 'ON'
106-
shell: bash
107-
run: ./faiss/gpu/hipify.sh
108104
- name: Symblink system dependencies
109105
if: inputs.raft == 'ON' || inputs.rocm == 'ON'
110106
shell: bash

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ if(FAISS_ENABLE_GPU)
7272
find_package(HIP REQUIRED)
7373
find_package(hipBLAS REQUIRED)
7474
set(GPU_EXT_PREFIX "hip")
75+
execute_process(COMMAND ${PROJECT_SOURCE_DIR}/faiss/gpu/hipify.sh)
7576
else ()
7677
set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER})
7778
enable_language(CUDA)

‎INSTALL.md‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -147,9 +147,7 @@ Several options can be passed to CMake, among which:
147147
to build against (see [CUDA docs](https://developer.nvidia.com/cuda-gpus) to
148148
determine which architecture(s) you should pick),
149149
- `-DFAISS_ENABLE_ROCM=ON` in order to enable building GPU indices for AMD GPUs.
150-
The hipify script must be executed before using this option.
151-
Invoke `./faiss/gpu/hipify.sh` to execute. `-DFAISS_ENABLE_GPU` must be `ON`
152-
when using this option. (possible values are `ON` and `OFF`),
150+
`-DFAISS_ENABLE_GPU` must be `ON` when using this option. (possible values are `ON` and `OFF`),
153151
- python-related options:
154152
- `-DPython_EXECUTABLE=/path/to/python3.7` in order to build a python
155153
interface for a different python than the default one (see

‎faiss/gpu/hipify.sh‎

Lines changed: 93 additions & 173 deletions
Original file line numberDiff line numberDiff line change
@@ -4,101 +4,87 @@
44
# This source code is licensed under the MIT license found in the
55
# LICENSE file in the root directory of this source tree.
66

7-
# go one level up from faiss/gpu
8-
top=$(dirname "${BASH_SOURCE[0]}")/..
9-
echo "top=$top"
10-
cd "$top" || exit
11-
echo "pwd=$(pwd)"
7+
function hipify_dir()
8+
{
9+
# print dir name
10+
cd "$1" || exit
11+
echo "Hipifying $(pwd)"
1212

13-
# create all destination directories for hipified files into sibling 'gpu-rocm' directory
14-
while IFS= read -r -d '' src
15-
do
16-
dst="${src//gpu/gpu-rocm}"
17-
echo "Creating $dst"
18-
mkdir -p "$dst"
19-
done < <(find ./gpu -type d -print0)
20-
21-
# run hipify-perl against all *.cu *.cuh *.h *.cpp files, no renaming
22-
# run all files in parallel to speed up
23-
for ext in cu cuh h cpp
24-
do
13+
# create all destination directories for hipified files into sibling 'gpu-rocm' directory
2514
while IFS= read -r -d '' src
2615
do
27-
dst="${src//\.\/gpu/\.\/gpu-rocm}"
28-
hipify-perl -o="$dst.tmp" "$src" &
29-
done < <(find ./gpu -name "*.$ext" -print0)
30-
done
31-
wait
16+
dst="${src//gpu/gpu-rocm}"
3217

33-
# rename all hipified *.cu files to *.hip
34-
while IFS= read -r -d '' src
35-
do
36-
dst=${src%.cu.tmp}.hip.tmp
37-
mv "$src" "$dst"
38-
done < <(find ./gpu-rocm -name "*.cu.tmp" -print0)
18+
if [ -d $dst ]; then
19+
#Clearing out any leftover files and directories
20+
echo "Removing old $dst"
21+
rm -rf "$dst"
22+
fi
3923

40-
# replace header include statements "<faiss/gpu/" with "<faiss/gpu-rocm"
41-
# replace thrust::cuda::par with thrust::hip::par
42-
# adjust header path location for hipblas.h to avoid unnecessary deprecation warnings
43-
# adjust header path location for hiprand_kernel.h to avoid unnecessary deprecation warnings
44-
for ext in hip cuh h cpp
45-
do
46-
while IFS= read -r -d '' src
47-
do
48-
sed -i 's@#include <faiss/gpu/@#include <faiss/gpu-rocm/@' "$src"
49-
sed -i 's@thrust::cuda::par@thrust::hip::par@' "$src"
50-
sed -i 's@#include <hipblas.h>@#include <hipblas/hipblas.h>@' "$src"
51-
sed -i 's@#include <hiprand_kernel.h>@#include <hiprand/hiprand_kernel.h>@' "$src"
52-
done < <(find ./gpu-rocm -name "*.$ext.tmp" -print0)
53-
done
24+
#Making directories
25+
echo "Creating $dst"
26+
mkdir -p "$dst"
27+
done < <(find ./gpu -type d -print0)
5428

55-
# hipify was run in parallel above
56-
# don't copy the tmp file if it is unchanged
57-
for ext in hip cuh h cpp
58-
do
29+
# run hipify-perl against all *.cu *.cuh *.h *.cpp files, no renaming
30+
# run all files in parallel to speed up
31+
for ext in cu cuh h cpp c
32+
do
33+
while IFS= read -r -d '' src
34+
do
35+
dst="${src//\.\/gpu/\.\/gpu-rocm}"
36+
hipify-perl -o="$dst.tmp" "$src" &
37+
done < <(find ./gpu -name "*.$ext" -print0)
38+
done
39+
wait
40+
41+
# rename all hipified *.cu files to *.hip
5942
while IFS= read -r -d '' src
6043
do
61-
dst=${src%.tmp}
62-
if test -f "$dst"
63-
then
64-
if diff -q "$src" "$dst" >& /dev/null
44+
dst=${src%.cu.tmp}.hip.tmp
45+
mv "$src" "$dst"
46+
done < <(find ./gpu-rocm -name "*.cu.tmp" -print0)
47+
48+
# replace header include statements "<faiss/gpu/" with "<faiss/gpu-rocm"
49+
# replace thrust::cuda::par with thrust::hip::par
50+
# adjust header path location for hipblas.h to avoid unnecessary deprecation warnings
51+
# adjust header path location for hiprand_kernel.h to avoid unnecessary deprecation warnings
52+
for ext in hip cuh h cpp c
53+
do
54+
while IFS= read -r -d '' src
55+
do
56+
sed -i 's@#include <faiss/gpu/@#include <faiss/gpu-rocm/@' "$src"
57+
sed -i 's@thrust::cuda::par@thrust::hip::par@' "$src"
58+
sed -i 's@#include <hipblas.h>@#include <hipblas/hipblas.h>@' "$src"
59+
sed -i 's@#include <hiprand_kernel.h>@#include <hiprand/hiprand_kernel.h>@' "$src"
60+
done < <(find ./gpu-rocm -name "*.$ext.tmp" -print0)
61+
done
62+
63+
# hipify was run in parallel above
64+
# don't copy the tmp file if it is unchanged
65+
for ext in hip cuh h cpp c
66+
do
67+
while IFS= read -r -d '' src
68+
do
69+
dst=${src%.tmp}
70+
if test -f "$dst"
6571
then
66-
echo "$dst [unchanged]"
67-
rm "$src"
72+
if diff -q "$src" "$dst" >& /dev/null
73+
then
74+
echo "$dst [unchanged]"
75+
rm "$src"
76+
else
77+
echo "$dst"
78+
mv "$src" "$dst"
79+
fi
6880
else
6981
echo "$dst"
7082
mv "$src" "$dst"
7183
fi
72-
else
73-
echo "$dst"
74-
mv "$src" "$dst"
75-
fi
76-
done < <(find ./gpu-rocm -name "*.$ext.tmp" -print0)
77-
done
78-
79-
# copy over CMakeLists.txt
80-
while IFS= read -r -d '' src
81-
do
82-
dst="${src//\.\/gpu/\.\/gpu-rocm}"
83-
if test -f "$dst"
84-
then
85-
if diff -q "$src" "$dst" >& /dev/null
86-
then
87-
echo "$dst [unchanged]"
88-
else
89-
echo "$dst"
90-
cp "$src" "$dst"
91-
fi
92-
else
93-
echo "$dst"
94-
cp "$src" "$dst"
95-
fi
96-
done < <(find ./gpu -name "CMakeLists.txt" -print0)
84+
done < <(find ./gpu-rocm -name "*.$ext.tmp" -print0)
85+
done
9786

98-
# Copy over other files
99-
other_exts="py"
100-
for ext in $other_exts
101-
do
87+
# copy over CMakeLists.txt
10288
while IFS= read -r -d '' src
10389
do
10490
dst="${src//\.\/gpu/\.\/gpu-rocm}"
@@ -115,102 +101,36 @@ do
115101
echo "$dst"
116102
cp "$src" "$dst"
117103
fi
118-
done < <(find ./gpu -name "*.$ext" -print0)
119-
done
120-
121-
###################################################################################
122-
# C_API Support
123-
###################################################################################
124-
125-
# Now get the c_api dir
126-
# This points to the faiss/c_api dir
127-
top_c_api=$(dirname "${BASH_SOURCE[0]}")/../../c_api
128-
echo "top=$top_c_api"
129-
cd "../$top_c_api" || exit
130-
echo "pwd=$(pwd)"
131-
132-
133-
# create all destination directories for hipified files into sibling 'gpu-rocm' directory
134-
while IFS= read -r -d '' src
135-
do
136-
dst="${src//gpu/gpu-rocm}"
137-
echo "Creating $dst"
138-
mkdir -p "$dst"
139-
done < <(find ./gpu -type d -print0)
104+
done < <(find ./gpu -name "CMakeLists.txt" -print0)
140105

141-
# run hipify-perl against all *.cu *.cuh *.h *.cpp files, no renaming
142-
# run all files in parallel to speed up
143-
for ext in cu cuh h cpp c
144-
do
145-
while IFS= read -r -d '' src
146-
do
147-
dst="${src//\.\/gpu/\.\/gpu-rocm}"
148-
hipify-perl -o="$dst.tmp" "$src" &
149-
done < <(find ./gpu -name "*.$ext" -print0)
150-
done
151-
wait
152-
153-
# rename all hipified *.cu files to *.hip
154-
while IFS= read -r -d '' src
155-
do
156-
dst=${src%.cu.tmp}.hip.tmp
157-
mv "$src" "$dst"
158-
done < <(find ./gpu-rocm -name "*.cu.tmp" -print0)
159-
160-
# replace header include statements "<faiss/gpu/" with "<faiss/gpu-rocm"
161-
# replace thrust::cuda::par with thrust::hip::par
162-
# adjust header path location for hipblas.h to avoid unnecessary deprecation warnings
163-
# adjust header path location for hiprand_kernel.h to avoid unnecessary deprecation warnings
164-
for ext in hip cuh h cpp c
165-
do
166-
while IFS= read -r -d '' src
106+
# Copy over other files
107+
other_exts="py"
108+
for ext in $other_exts
167109
do
168-
sed -i 's@#include <faiss/gpu/@#include <faiss/gpu-rocm/@' "$src"
169-
sed -i 's@thrust::cuda::par@thrust::hip::par@' "$src"
170-
sed -i 's@#include <hipblas.h>@#include <hipblas/hipblas.h>@' "$src"
171-
sed -i 's@#include <hiprand_kernel.h>@#include <hiprand/hiprand_kernel.h>@' "$src"
172-
done < <(find ./gpu-rocm -name "*.$ext.tmp" -print0)
173-
done
174-
175-
# hipify was run in parallel above
176-
# don't copy the tmp file if it is unchanged
177-
for ext in hip cuh h cpp c
178-
do
179-
while IFS= read -r -d '' src
180-
do
181-
dst=${src%.tmp}
182-
if test -f "$dst"
183-
then
184-
if diff -q "$src" "$dst" >& /dev/null
110+
while IFS= read -r -d '' src
111+
do
112+
dst="${src//\.\/gpu/\.\/gpu-rocm}"
113+
if test -f "$dst"
185114
then
186-
echo "$dst [unchanged]"
187-
rm "$src"
115+
if diff -q "$src" "$dst" >& /dev/null
116+
then
117+
echo "$dst [unchanged]"
118+
else
119+
echo "$dst"
120+
cp "$src" "$dst"
121+
fi
188122
else
189123
echo "$dst"
190-
mv "$src" "$dst"
124+
cp "$src" "$dst"
191125
fi
192-
else
193-
echo "$dst"
194-
mv "$src" "$dst"
195-
fi
196-
done < <(find ./gpu-rocm -name "*.$ext.tmp" -print0)
197-
done
126+
done < <(find ./gpu -name "*.$ext" -print0)
127+
done
128+
}
198129

199-
# copy over CMakeLists.txt
200-
while IFS= read -r -d '' src
201-
do
202-
dst="${src//\.\/gpu/\.\/gpu-rocm}"
203-
if test -f "$dst"
204-
then
205-
if diff -q "$src" "$dst" >& /dev/null
206-
then
207-
echo "$dst [unchanged]"
208-
else
209-
echo "$dst"
210-
cp "$src" "$dst"
211-
fi
212-
else
213-
echo "$dst"
214-
cp "$src" "$dst"
215-
fi
216-
done < <(find ./gpu -name "CMakeLists.txt" -print0)
130+
# Convert the faiss/gpu dir
131+
dir_name=$(dirname "${BASH_SOURCE[0]}")/..
132+
hipify_dir $dir_name
133+
134+
# Convert the faiss/c_api dir
135+
dir_name=$(dirname "${BASH_SOURCE[0]}")/../../c_api
136+
hipify_dir $dir_name

0 commit comments

Comments
 (0)