Skip to content

Commit da84da7

Browse files
czr-gcczr-gc
authored
[Serving]: add ipu support for serving. (#10) (#470)
* feat(ipu): add ipu docker for serving. (#10) * feat(ipu): add ipu docker for serving. * feat(ipu): enable ipu docker in serving. * fix(): fix typo and issues in IPU. * remove unused env path. * doc(ipu): add ipu docker build doc and fix typo. * fix(): clean apt cache in docker. * fix(ipu): fix typo.
1 parent 5a1059c commit da84da7

File tree

4 files changed

+149
-3
lines changed

4 files changed

+149
-3
lines changed

‎serving/Dockerfile_ipu‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
FROM graphcore/poplar:3.0.0
16+
17+
#Install the build dependencies
18+
RUN apt-get update && apt-get install -y --no-install-recommends curl wget vim git patchelf python3-dev python3-pip \
19+
python3-setuptools build-essential libgl1-mesa-glx libglib2.0-dev ca-certificates \
20+
libssl-dev zlib1g-dev rapidjson-dev libboost-dev libre2-dev librdmacm-dev libnuma-dev libarchive-dev unzip && \
21+
apt-get clean && rm -rf /var/lib/apt/lists/*
22+
23+
RUN ln -s /usr/bin/python3 /usr/bin/python;
24+
RUN pip3 install --upgrade pip
25+
26+
# install cmake
27+
WORKDIR /home
28+
RUN wget -q https://github.com/Kitware/CMake/releases/download/v3.18.6/cmake-3.18.6-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.6-Linux-x86_64.tar.gz
29+
ENV PATH=/home/cmake-3.18.6-Linux-x86_64/bin:$PATH
30+
31+
32+
#install triton
33+
ENV TAG=r21.10
34+
RUN git clone https://github.com/triton-inference-server/server.git -b $TAG && \
35+
cd server && \
36+
mkdir -p build/tritonserver/install && \
37+
python3 build.py \
38+
--build-dir `pwd`/build \
39+
--no-container-build \
40+
--endpoint=grpc \
41+
--enable-logging \
42+
--enable-stats \
43+
--cmake-dir `pwd`/build \
44+
--repo-tag=common:$TAG \
45+
--repo-tag=core:$TAG \
46+
--repo-tag=backend:$TAG \
47+
--repo-tag=thirdparty:$TAG \
48+
--backend=python:$TAG
49+
50+
COPY python/dist/*.whl /opt/fastdeploy/
51+
RUN python3 -m pip install /opt/fastdeploy/*.whl \
52+
&& rm -rf /opt/fastdeploy/*.whl
53+
54+
# triton server
55+
RUN mkdir -p /opt/tritonserver && cp -r /home/server/build/tritonserver/install/* /opt/tritonserver
56+
# python backend
57+
RUN mkdir -p /opt/tritonserver/backends/python && cp -r /home/server/build/python/install/backends/python /opt/tritonserver/backends/
58+
# fd_backend
59+
COPY serving/build/libtriton_fastdeploy.so /opt/tritonserver/backends/fastdeploy/
60+
61+
COPY build/fastdeploy-0.0.3 /opt/fastdeploy/
62+
RUN mv /opt/tritonserver/bin/tritonserver /opt/tritonserver/bin/fastdeployserver
63+
ENV LD_LIBRARY_PATH="/opt/fastdeploy/lib:/opt/fastdeploy/third_libs/install/onnxruntime/lib:/opt/fastdeploy/third_libs/install/paddle2onnx/lib:/opt/fastdeploy/third_libs/install/paddle_inference/paddle/lib:$LD_LIBRARY_PATH"
64+
ENV PATH="/opt/tritonserver/bin:$PATH"

‎serving/docs/zh_CN/compile.md‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
FastDploy发布的GPU镜像基于[Triton Inference Server](https://github.com/triton-inference-server/server)的21.10版本进行制作，如果有其他CUDA版本需求，可以参照[NVIDIA 官网](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html)中展示的版本信息修改Dockerfile和scripts中的脚本.
88

99
```
10-
# 进入serving目录执行脚本编译fastdeply和服务化的backend
10+
# 进入serving目录执行脚本编译fastdeploy和服务化的backend
1111
cd serving
1212
bash scripts/build.sh
1313
@@ -19,11 +19,23 @@ docker build -t paddlepaddle/fastdeploy:0.3.0-gpu-cuda11.4-trt8.4-21.10 -f servi
1919
## 制作CPU镜像
2020

2121
```
22-
# 进入serving目录执行脚本编译fastdeply和服务化的backend
22+
# 进入serving目录执行脚本编译fastdeploy和服务化的backend
2323
cd serving
2424
bash scripts/build.sh OFF
2525
2626
# 退出到FastDeploy主目录，制作镜像
2727
cd ../
2828
docker build -t paddlepaddle/fastdeploy:0.3.0-cpu-only-21.10 -f serving/Dockerfile_cpu .
2929
```
30+
31+
## 制作IPU镜像
32+
33+
```
34+
# 进入serving目录执行脚本编译fastdeploy和服务化的backend
35+
cd serving
36+
bash scripts/build_fd_ipu.sh
37+
38+
# 退出到FastDeploy主目录，制作镜像
39+
cd ../
40+
docker build -t paddlepaddle/fastdeploy:0.3.0-ipu-only-21.10 -f serving/Dockerfile_ipu .
41+
```

‎serving/scripts/build_fd_ipu.sh‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
#!/usr/bin/env bash
2+
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
if [ ! -d "./cmake-3.18.6-Linux-x86_64/" ]; then
17+
wget https://github.com/Kitware/CMake/releases/download/v3.18.6/cmake-3.18.6-Linux-x86_64.tar.gz
18+
tar -zxvf cmake-3.18.6-Linux-x86_64.tar.gz
19+
rm -rf cmake-3.18.6-Linux-x86_64.tar.gz
20+
fi
21+
22+
# build vision
23+
docker run -it --rm --name build_fd_vison \
24+
-v`pwd`/..:/workspace/fastdeploy \
25+
graphcore/poplar:3.0.0 \
26+
bash -c \
27+
'cd /workspace/fastdeploy/python;
28+
rm -rf .setuptools-cmake-build dist;
29+
apt-get update;
30+
apt-get install -y --no-install-recommends patchelf python3-dev python3-pip python3-setuptools build-essential;
31+
ln -s /usr/bin/python3 /usr/bin/python;
32+
pip3 install wheel;
33+
export PATH=/workspace/fastdeploy/serving/cmake-3.18.6-Linux-x86_64/bin:$PATH;
34+
export WITH_GPU=OFF;
35+
export WITH_IPU=ON;
36+
export ENABLE_PADDLE_BACKEND=ON;
37+
export ENABLE_VISION=ON;
38+
python setup.py build;
39+
python setup.py bdist_wheel'
40+
41+
# build runtime
42+
docker run -it --rm --name build_fd_runtime \
43+
-v`pwd`/..:/workspace/fastdeploy \
44+
graphcore/poplar:3.0.0 \
45+
bash -c \
46+
'cd /workspace/fastdeploy;
47+
rm -rf build; mkdir build; cd build;
48+
apt-get update;
49+
apt-get install -y --no-install-recommends python3-dev python3-pip build-essential;
50+
ln -s /usr/bin/python3 /usr/bin/python;
51+
export PATH=/workspace/fastdeploy/serving/cmake-3.18.6-Linux-x86_64/bin:$PATH;
52+
cmake .. -DENABLE_ORT_BACKEND=OFF -DENABLE_TEXT=OFF -DENABLE_VISION=OFF -DBUILD_FASTDEPLOY_PYTHON=OFF -DENABLE_PADDLE_BACKEND=ON -DWITH_IPU=ON -DCMAKE_INSTALL_PREFIX=${PWD}/fastdeploy-0.0.3 -DLIBRARY_NAME=fastdeploy_runtime;
53+
make -j`nproc`;
54+
make install'
55+
56+
# build backend
57+
docker run -it --rm --name build_fd_backend \
58+
-v`pwd`/..:/workspace/fastdeploy \
59+
graphcore/poplar:3.0.0 \
60+
bash -c \
61+
'cd /workspace/fastdeploy/serving;
62+
rm -rf build; mkdir build; cd build;
63+
apt-get update; apt-get install -y --no-install-recommends rapidjson-dev build-essential git ca-certificates;
64+
export PATH=/workspace/fastdeploy/serving/cmake-3.18.6-Linux-x86_64/bin:$PATH;
65+
cmake .. -DTRITON_ENABLE_GPU=OFF -DFASTDEPLOY_DIR=/workspace/fastdeploy/build/fastdeploy-0.0.3 -DTRITON_COMMON_REPO_TAG=r21.10 -DTRITON_CORE_REPO_TAG=r21.10 -DTRITON_BACKEND_REPO_TAG=r21.10; make -j`nproc`'

‎serving/src/fastdeploy_runtime.cc‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,8 @@ ModelState::ModelState(TRITONBACKEND_Model* triton_model)
238238
runtime_options_->SetPaddleMKLDNN(pd_enable_mkldnn);
239239
} else if (param_key == "use_paddle_log") {
240240
runtime_options_->EnablePaddleLogInfo();
241+
} else if (param_key == "use_ipu") {
242+
runtime_options_->UseIpu();
241243
}
242244
}
243245
}
@@ -384,7 +386,10 @@ TRITONSERVER_Error* ModelState::LoadModel(
384386
runtime_options_->UseCpu();
385387
}
386388
#else
387-
runtime_options_->UseCpu();
389+
if (runtime_options_->device != fastdeploy::Device::IPU) {
390+
// If Device is set to IPU, just skip CPU setting.
391+
runtime_options_->UseCpu();
392+
}
388393
#endif // TRITON_ENABLE_GPU
389394

390395
*runtime = new fastdeploy::Runtime();

0 commit comments

Comments
 (0)