Add Paddle Inference as backend

CMakeLists.txt

@@ -29,6 +29,7 @@ option(ENABLE_PADDLE_FRONTEND "if to enable PaddlePaddle frontend to support loa
 option(WITH_GPU "if WITH_GPU=ON, will enable onnxruntime-gpu/paddle-infernce-gpu" OFF)
 option(ENABLE_ORT_BACKEND "if to enable onnxruntime backend." OFF)
 option(ENABLE_TRT_BACKEND "if to enable tensorrt backend." OFF)
+option(ENABLE_PADDLE_BACKEND "if to enable paddle backend." OFF)
 option(CUDA_DIRECTORY "if build tensorrt backend, need to define path of cuda library.")
 option(TRT_DIRECTORY "if build tensorrt backend, need to define path of tensorrt library.")
 option(ENABLE_VISION "if to enable vision models usage." OFF)
@@ -39,7 +40,7 @@ option(ENABLE_OPENCV_CUDA "if to enable opencv with cuda, this will allow proces
 option(ENABLE_DEBUG "if to enable print debug information, this may reduce performance." OFF)
 
 # Whether to build fastdeply with vision/text/... examples, only for testings.
-option(WITH_VISION_EXAMPLES "Whether to build fastdeply with vision examples" ON)
+option(WITH_VISION_EXAMPLES "Whether to build fastdeply with vision examples" OFF)
 
 if(ENABLE_DEBUG)
   add_definitions(-DFASTDEPLOY_DEBUG)
@@ -63,10 +64,11 @@ endif()
 add_definitions(-DFASTDEPLOY_LIB)
 file(GLOB_RECURSE ALL_DEPLOY_SRCS ${PROJECT_SOURCE_DIR}/fastdeploy/*.cc)
 file(GLOB_RECURSE DEPLOY_ORT_SRCS ${PROJECT_SOURCE_DIR}/fastdeploy/backends/ort/*.cc)
+file(GLOB_RECURSE DEPLOY_PADDLE_SRCS ${PROJECT_SOURCE_DIR}/fastdeploy/backends/paddle/*.cc)
 file(GLOB_RECURSE DEPLOY_TRT_SRCS ${PROJECT_SOURCE_DIR}/fastdeploy/backends/tensorrt/*.cc ${PROJECT_SOURCE_DIR}/fastdeploy/backends/tensorrt/*.cpp)
 file(GLOB_RECURSE DEPLOY_VISION_SRCS ${PROJECT_SOURCE_DIR}/fastdeploy/vision/*.cc)
 file(GLOB_RECURSE DEPLOY_PYBIND_SRCS ${PROJECT_SOURCE_DIR}/fastdeploy/pybind/*.cc ${PROJECT_SOURCE_DIR}/fastdeploy/*_pybind.cc)
-list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_VISION_SRCS})
+list(REMOVE_ITEM ALL_DEPLOY_SRCS ${DEPLOY_ORT_SRCS} ${DEPLOY_PADDLE_SRCS} ${DEPLOY_TRT_SRCS} ${DEPLOY_VISION_SRCS})
 
 set(DEPEND_LIBS "")
 
@@ -87,6 +89,13 @@ if(ENABLE_ORT_BACKEND)
   list(APPEND DEPEND_LIBS external_onnxruntime)
 endif()
 
+if(ENABLE_PADDLE_BACKEND)
+  add_definitions(-DENABLE_PADDLE_BACKEND)
+  list(APPEND ALL_DEPLOY_SRCS ${DEPLOY_PADDLE_SRCS})
+  include(external/paddle_inference.cmake)
+  list(APPEND DEPEND_LIBS external_paddle_inference external_dnnl external_omp)
+endif()
+
 if(WITH_GPU)
   if(APPLE)
     message(FATAL_ERROR "Cannot enable GPU while compling in Mac OSX.")
@@ -280,3 +289,10 @@ if(BUILD_FASTDEPLOY_PYTHON)
   endif()
 
 endif(BUILD_FASTDEPLOY_PYTHON)
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.4.0")
+    string(STRIP "${CMAKE_CXX_COMPILER_VERSION}" CMAKE_CXX_COMPILER_VERSION)
+    message(WARNING "[WARNING] FastDeploy require g++ version >= 5.4.0, but now your g++ version is ${CMAKE_CXX_COMPILER_VERSION}, this may cause failure! Use -DCMAKE_CXX_COMPILER to define path of your compiler.")
+  endif()
+endif()

FastDeploy.cmake.in

@@ -2,7 +2,9 @@ CMAKE_MINIMUM_REQUIRED (VERSION 3.16)
 
 set(WITH_GPU @WITH_GPU@)
 set(ENABLE_ORT_BACKEND @ENABLE_ORT_BACKEND@)
-set(ENABLE_TRT_BACKEND @ENABLE_TRT_BACKEND&)
+set(ENABLE_PADDLE_BACKEND @ENABLE_PADDLE_BACKEND@)
+set(PADDLEINFERENCE_VERSION @PADDLEINFERENCE_VERSION@)
+set(ENABLE_TRT_BACKEND @ENABLE_TRT_BACKEND@)
 set(ENABLE_PADDLE_FRONTEND @ENABLE_PADDLE_FRONTEND@)
 set(ENABLE_VISION @ENABLE_VISION@)
 set(ENABLE_OPENCV_CUDA @ENABLE_OPENCV_CUDA@)
@@ -17,16 +19,35 @@ endif()
 
 find_library(FDLIB fastdeploy ${CMAKE_CURRENT_LIST_DIR}/lib)
 list(APPEND FASTDEPLOY_LIBS ${FDLIB})
+
 if(ENABLE_ORT_BACKEND)
   find_library(ORT_LIB onnxruntime ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/onnxruntime/lib)
   list(APPEND FASTDEPLOY_LIBS ${ORT_LIB})
 endif()
 
+if(ENABLE_PADDLE_BACKEND)
+  find_library(PADDLE_LIB paddle_inference ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/paddle/lib)
+  if(WIN32)
+    set(DNNL_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mkldnn/lib/mkldnn.lib")
+    set(IOMP_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mklml/lib/libiomp5.lib")
+  elseif(APPLE)
+    set(DNNL_LIB "")
+    set(IOMP_LIB "")
+  else()
+    set(DNNL_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mkldnn/lib/libmkldnn.so.0")
+    set(IOMP_LIB "${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle_inference/third_party/install/mklml/lib/libiomp5.so")
+  endif()
+  list(APPEND FASTDEPLOY_LIBS ${PADDLE_LIB} ${DNNL_LIB} ${IOMP_LIB})
+endif()
+
 if(WITH_GPU)
   if (NOT CUDA_DIRECTORY)
-    message(FATAL_ERROR "[FastDeploy] Please define CUDA_DIRECTORY, e.g -DCUDA_DIRECTORY=/usr/local/cuda")
+    set(CUDA_DIRECTORY "/usr/local/cuda")
   endif()
   find_library(CUDA_LIB cudart ${CUDA_DIRECTORY}/lib64)
+  if(NOT CUDA_LIB)
+    message(FATAL_ERROR "[FastDeploy] Cannot find library cudart in ${CUDA_DIRECTORY}, Please define CUDA_DIRECTORY, e.g -DCUDA_DIRECTORY=/path/to/cuda")
+  endif()
   list(APPEND FASTDEPLOY_LIBS ${CUDA_LIB})
 
   if (ENABLE_TRT_BACKEND)
@@ -61,3 +82,28 @@ if(ENABLE_PADDLE_FRONTEND)
   find_library(PADDLE2ONNX_LIB paddle2onnx  ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/paddle2onnx/lib)
   list(APPEND FASTDEPLOY_LIBS ${PADDLE2ONNX_LIB})
 endif()
+
+# Print compiler information
+message(STATUS "")
+message(STATUS "*************FastDeploy Building Summary**********")
+message(STATUS "  CMake version             : ${CMAKE_VERSION}")
+message(STATUS "  CMake command             : ${CMAKE_COMMAND}")
+message(STATUS "  System                    : ${CMAKE_SYSTEM_NAME}")
+message(STATUS "  C++ compiler              : ${CMAKE_CXX_COMPILER}")
+message(STATUS "  C++ compiler version      : ${CMAKE_CXX_COMPILER_VERSION}")
+message(STATUS "  CXX flags                 : ${CMAKE_CXX_FLAGS}")
+message(STATUS "  WITH_GPU                  : ${WITH_GPU}")
+message(STATUS "  ENABLE_ORT_BACKEND        : ${ENABLE_ORT_BACKEND}")
+message(STATUS "  ENABLE_PADDLE_BACKEND     : ${ENABLE_PADDLE_BACKEND}")
+if(ENABLE_PADDLE_BACKEND)
+  message(STATUS "  Paddle Inference version  : ${PADDLEINFERENCE_VERSION}")
+endif()
+message(STATUS "  ENABLE_TRT_BACKEND        : ${ENABLE_TRT_BACKEND}")
+message(STATUS "  ENABLE_VISION             : ${ENABLE_VISION}")
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.4.0")
+    string(STRIP "${CMAKE_CXX_COMPILER_VERSION}" CMAKE_CXX_COMPILER_VERSION)
+    message(WARNING "[WARNING] FastDeploy require g++ version >= 5.4.0, but now your g++ version is ${CMAKE_CXX_COMPILER_VERSION}, this may cause failure! Use -DCMAKE_CXX_COMPILER to define path of your compiler.")
+  endif()
+endif()

external/paddle_inference.cmake

@@ -0,0 +1,107 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+include(ExternalProject)
+
+set(PADDLEINFERENCE_PROJECT "extern_paddle_inference")
+set(PADDLEINFERENCE_PREFIX_DIR ${THIRD_PARTY_PATH}/paddle_inference)
+set(PADDLEINFERENCE_SOURCE_DIR
+    ${THIRD_PARTY_PATH}/paddle_inference/src/${PADDLEINFERENCE_PROJECT})
+set(PADDLEINFERENCE_INSTALL_DIR ${THIRD_PARTY_PATH}/install/paddle_inference)
+set(PADDLEINFERENCE_INC_DIR
+    "${PADDLEINFERENCE_INSTALL_DIR}/paddle/include"
+    CACHE PATH "paddle_inference include directory." FORCE)
+set(PADDLEINFERENCE_LIB_DIR
+    "${PADDLEINFERENCE_INSTALL_DIR}/paddle/lib/"
+    CACHE PATH "paddle_inference lib directory." FORCE)
+set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}"
+                      "${PADDLEINFERENCE_LIB_DIR}")
+
+include_directories(${PADDLEINFERENCE_INC_DIR})
+if(WIN32)
+  set(PADDLEINFERENCE_COMPILE_LIB
+      "${PADDLEINFERENCE_INSTALL_DIR}/paddle/lib/paddle_inference.lib"
+      CACHE FILEPATH "paddle_inference compile library." FORCE)
+  set(DNNL_LIB "")
+  set(OMP_LIB "")
+elseif(APPLE)
+  set(PADDLEINFERENCE_COMPILE_LIB
+      "${PADDLEINFERENCE_INSTALL_DIR}/paddle/lib/libpaddle_inference.dylib"
+      CACHE FILEPATH "paddle_inference compile library." FORCE)
+  set(DNNL_LIB "")
+  set(OMP_LIB "")
+else()
+  set(PADDLEINFERENCE_COMPILE_LIB
+      "${PADDLEINFERENCE_INSTALL_DIR}/paddle/lib/libpaddle_inference.so"
+      CACHE FILEPATH "paddle_inference compile library." FORCE)
+  set(DNNL_LIB "${PADDLEINFERENCE_INSTALL_DIR}/third_party/install/mkldnn/lib/libdnnl.so.2")
+  set(OMP_LIB "${PADDLEINFERENCE_INSTALL_DIR}/third_party/install/mklml/lib/libiomp5.so")
+endif(WIN32)
+
+set(PADDLEINFERENCE_URL_BASE "https://bj.bcebos.com/paddle2onnx/libs/")
+set(PADDLEINFERENCE_VERSION "2.3.1")
+if(WIN32)
+  message(FATAL_ERROR "Paddle Backend doesn't support Windows now.")
+  set(PADDLEINFERENCE_FILE "paddle_inference-win-x64-${PADDLEINFERENCE_VERSION}.zip")
+elseif(APPLE)
+  message(FATAL_ERROR "Paddle Backend doesn't support Mac OSX now.")
+  if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64")
+    set(PADDLEINFERENCE_FILE "paddle_inference-osx-arm64-${PADDLEINFERENCE_VERSION}.tgz")
+  else()
+    set(PADDLEINFERENCE_FILE "paddle_inference-osx-x86_64-${PADDLEINFERENCE_VERSION}.tgz")
+  endif()
+else()
+  if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
+    message(FATAL_ERROR "Paddle Backend doesn't support linux aarch64 now.")
+    set(PADDLEINFERENCE_FILE "paddle_inference-linux-aarch64-${PADDLEINFERENCE_VERSION}.tgz")
+  else()
+    set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-${PADDLEINFERENCE_VERSION}.tgz")
+    if(WITH_GPU)
+        set(PADDLEINFERENCE_FILE "paddle_inference-linux-x64-gpu-${PADDLEINFERENCE_VERSION}.tgz")
+    endif()
+  endif()
+endif()
+set(PADDLEINFERENCE_URL "${PADDLEINFERENCE_URL_BASE}${PADDLEINFERENCE_FILE}")
+
+ExternalProject_Add(
+  ${PADDLEINFERENCE_PROJECT}
+  ${EXTERNAL_PROJECT_LOG_ARGS}
+  URL ${PADDLEINFERENCE_URL}
+  PREFIX ${PADDLEINFERENCE_PREFIX_DIR}
+  DOWNLOAD_NO_PROGRESS 1
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND ""
+  UPDATE_COMMAND ""
+  INSTALL_COMMAND
+    ${CMAKE_COMMAND} -E remove_directory ${PADDLEINFERENCE_INSTALL_DIR} &&
+    ${CMAKE_COMMAND} -E make_directory ${PADDLEINFERENCE_INSTALL_DIR} &&
+    ${CMAKE_COMMAND} -E rename ${PADDLEINFERENCE_SOURCE_DIR}/paddle/
+    ${PADDLEINFERENCE_INSTALL_DIR}/paddle && ${CMAKE_COMMAND} -E rename 
+    ${PADDLEINFERENCE_SOURCE_DIR}/third_party ${PADDLEINFERENCE_INSTALL_DIR}/third_party &&
+    ${CMAKE_COMMAND} -E rename ${PADDLEINFERENCE_SOURCE_DIR}/version.txt ${PADDLEINFERENCE_INSTALL_DIR}/version.txt
+  BUILD_BYPRODUCTS ${PADDLEINFERENCE_COMPILE_LIB})
+
+add_library(external_paddle_inference STATIC IMPORTED GLOBAL)
+set_property(TARGET external_paddle_inference PROPERTY IMPORTED_LOCATION
+                                         ${PADDLEINFERENCE_COMPILE_LIB})
+add_dependencies(external_paddle_inference ${PADDLEINFERENCE_PROJECT})
+
+add_library(external_dnnl STATIC IMPORTED GLOBAL)
+set_property(TARGET external_dnnl PROPERTY IMPORTED_LOCATION
+                                         ${DNNL_LIB})
+add_dependencies(external_dnnl ${PADDLEINFERENCE_PROJECT})
+
+add_library(external_omp STATIC IMPORTED GLOBAL)
+set_property(TARGET external_omp PROPERTY IMPORTED_LOCATION
+                                         ${OMP_LIB})
+add_dependencies(external_omp ${PADDLEINFERENCE_PROJECT})

external/summary.cmake

@@ -30,11 +30,17 @@ function(fastdeploy_summary)
   message(STATUS "")
   message(STATUS "  FastDeploy version        : ${FASTDEPLOY_VERSION}")
   message(STATUS "  Paddle2ONNX version       : ${PADDLE2ONNX_VERSION}")
-  message(STATUS "  ONNXRuntime version       : ${ONNXRUNTIME_VERSION}")
   message(STATUS "  ENABLE_ORT_BACKEND        : ${ENABLE_ORT_BACKEND}")
+  message(STATUS "  ENABLE_PADDLE_BACKEND     : ${ENABLE_PADDLE_BACKEND}")
+  message(STATUS "  ENABLE_TRT_BACKEND        : ${ENABLE_TRT_BACKEND}")
+  if(ENABLE_ORT_BACKEND)
+    message(STATUS "  ONNXRuntime version       : ${ONNXRUNTIME_VERSION}")
+  endif()
+  if(ENABLE_PADDLE_BACKEND)
+    message(STATUS "  Paddle Inference version  : ${PADDLEINFERENCE_VERSION}")
+  endif()
   if(WITH_GPU)
     message(STATUS "  WITH_GPU                  : ${WITH_GPU}")
-    message(STATUS "  ENABLE_TRT_BACKEND        : ${ENABLE_TRT_BACKEND}")
     message(STATUS "  CUDA_DIRECTORY            : ${CUDA_DIRECTORY}")
     message(STATUS "  TRT_DRECTORY              : ${TRT_DIRECTORY}")
   endif()

fastdeploy/backends/paddle/paddle_backend.cc

@@ -0,0 +1,105 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "fastdeploy/backends/paddle/paddle_backend.h"
+
+namespace fastdeploy {
+
+void PaddleBackend::BuildOption(const PaddleBackendOption& option) {
+  if (option.use_gpu) {
+    config_.EnableUseGpu(option.gpu_mem_init_size, option.gpu_id);
+  } else {
+    config_.DisableGpu();
+    if (option.enable_mkldnn) {
+      config_.EnableMKLDNN();
+      config_.SetMkldnnCacheCapacity(option.mkldnn_cache_size);
+    }
+  }
+  config_.SetCpuMathLibraryNumThreads(option.cpu_thread_num);
+}
+
+bool PaddleBackend::InitFromPaddle(const std::string& model_file,
+                                   const std::string& params_file,
+                                   const PaddleBackendOption& option) {
+  if (initialized_) {
+    FDERROR << "PaddleBackend is already initlized, cannot initialize again."
+            << std::endl;
+    return false;
+  }
+  config_.SetModel(model_file, params_file);
+  BuildOption(option);
+  predictor_ = paddle_infer::CreatePredictor(config_);
+  std::vector<std::string> input_names = predictor_->GetInputNames();
+  std::vector<std::string> output_names = predictor_->GetOutputNames();
+  for (size_t i = 0; i < input_names.size(); ++i) {
+    auto handle = predictor_->GetInputHandle(input_names[i]);
+    TensorInfo info;
+    auto shape = handle->shape();
+    info.shape.assign(shape.begin(), shape.end());
+    info.dtype = PaddleDataTypeToFD(handle->type());
+    info.name = input_names[i];
+    inputs_desc_.emplace_back(info);
+  }
+  for (size_t i = 0; i < output_names.size(); ++i) {
+    auto handle = predictor_->GetOutputHandle(output_names[i]);
+    TensorInfo info;
+    auto shape = handle->shape();
+    info.shape.assign(shape.begin(), shape.end());
+    info.dtype = PaddleDataTypeToFD(handle->type());
+    info.name = output_names[i];
+    outputs_desc_.emplace_back(info);
+  }
+  initialized_ = true;
+  return true;
+}
+
+TensorInfo PaddleBackend::GetInputInfo(int index) {
+  FDASSERT(index < NumInputs(), "The index:" + std::to_string(index) +
+                                    " should less than the number of inputs:" +
+                                    std::to_string(NumInputs()) + ".");
+  return inputs_desc_[index];
+}
+
+TensorInfo PaddleBackend::GetOutputInfo(int index) {
+  FDASSERT(index < NumOutputs(),
+           "The index:" + std::to_string(index) +
+               " should less than the number of outputs:" +
+               std::to_string(NumOutputs()) + ".");
+  return outputs_desc_[index];
+}
+
+bool PaddleBackend::Infer(std::vector<FDTensor>& inputs,
+                          std::vector<FDTensor>* outputs) {
+  if (inputs.size() != inputs_desc_.size()) {
+    FDERROR << "[PaddleBackend] Size of inputs(" << inputs.size()
+            << ") should keep same with the inputs of this model("
+            << inputs_desc_.size() << ")." << std::endl;
+    return false;
+  }
+
+  for (size_t i = 0; i < inputs.size(); ++i) {
+    auto handle = predictor_->GetInputHandle(inputs[i].name);
+    ShareTensorFromCpu(handle.get(), inputs[i]);
+  }
+
+  predictor_->Run();
+  outputs->resize(outputs_desc_.size());
+  for (size_t i = 0; i < outputs_desc_.size(); ++i) {
+    auto handle = predictor_->GetOutputHandle(outputs_desc_[i].name);
+    CopyTensorToCpu(handle, &((*outputs)[i]));
+  }
+  return true;
+}
+
+}  // namespace fastdeploy