diff --git a/CMakeLists.txt b/CMakeLists.txt index 71aeb8165ce..b6ac77f4f76 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,11 +117,11 @@ if(ENABLE_TRT_BACKEND) list(APPEND DEPEND_LIBS ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_CAFFE_LIB} ${TRT_PLUGIN_LIB}) # copy tensorrt libraries to third lib - if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt") - file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib") - endif() - file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib") - file(COPY ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_CAFFE_LIB} ${TRT_PLUGIN_LIB} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib" FOLLOW_SYMLINK_CHAIN) +# if(EXISTS "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt") +# file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib") +# endif() +# file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib") +# file(COPY ${TRT_INFER_LIB} ${TRT_ONNX_LIB} ${TRT_CAFFE_LIB} ${TRT_PLUGIN_LIB} DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/third_libs/install/tensorrt/lib" FOLLOW_SYMLINK_CHAIN) endif() if(ENABLE_VISION) @@ -278,4 +278,5 @@ if(BUILD_FASTDEPLOY_PYTHON) ${EXTRA_FLAGS}) target_compile_options(fastdeploy_main PRIVATE $<$>:/MT> $<$:/MTd>) endif() + endif(BUILD_FASTDEPLOY_PYTHON) diff --git a/fastdeploy/fastdeploy_runtime.cc b/fastdeploy/fastdeploy_runtime.cc index 3141e71721e..28f363d8508 100644 --- a/fastdeploy/fastdeploy_runtime.cc +++ b/fastdeploy/fastdeploy_runtime.cc @@ -75,8 +75,10 @@ bool ModelFormatCheck(const std::string& model_file, bool Runtime::Init(const RuntimeOption& _option) { option = _option; if (option.backend == Backend::ORT) { + FDASSERT(option.device == Device::CPU || option.device == Device::GPU, "Backend::TRT only supports Device::CPU/Device::GPU."); CreateOrtBackend(); } else if (option.backend == Backend::TRT) { + FDASSERT(option.device == Device::GPU, "Backend::TRT only supports Device::GPU."); CreateTrtBackend(); } else { FDERROR << "Runtime only support Backend::ORT/Backend::TRT as backend now." diff --git a/fastdeploy/fastdeploy_runtime.py b/fastdeploy/fastdeploy_runtime.py index 4b7acb25edd..592d1d29529 100644 --- a/fastdeploy/fastdeploy_runtime.py +++ b/fastdeploy/fastdeploy_runtime.py @@ -53,3 +53,29 @@ def initialized(self): if self._model is None: return False return self._model.initialized() + + +class FastDeployRuntime: + def __init__(self, runtime_option): + self._runtime = C.Runtime(); + assert self._runtime.init(runtime_option), "Initialize FastDeployRuntime Failed!" + + def infer(self, data): + assert isinstance(data, dict), "The input data should be type of dict." + return self._runtime.infer(data) + + def num_inputs(self): + return self._runtime.num_inputs(); + + def num_outputs(self): + return self._runtime.num_outputs(); + + def get_input_info(self, index): + assert isinstance(index, int), "The input parameter index should be type of int." + assert index < self.num_inputs(), "The input parameter index:{} should less than number of inputs:{}.".format(index, self.num_inputs) + return self._runtime.get_input_info(index) + + def get_output_info(self, index): + assert isinstance(index, int), "The input parameter index should be type of int." + assert index < self.num_outputs(), "The input parameter index:{} should less than number of outputs:{}.".format(index, self.num_outputs) + return self._runtime.get_output_info(index) diff --git a/fastdeploy/pybind/fastdeploy_runtime.cc b/fastdeploy/pybind/fastdeploy_runtime.cc index 27174537d34..e3c6dd19ae2 100644 --- a/fastdeploy/pybind/fastdeploy_runtime.cc +++ b/fastdeploy/pybind/fastdeploy_runtime.cc @@ -40,12 +40,15 @@ void BindRuntime(pybind11::module& m) { .def_readwrite("trt_max_batch_size", &RuntimeOption::trt_max_batch_size) .def_readwrite("trt_max_workspace_size", &RuntimeOption::trt_max_workspace_size); + + pybind11::class_(m, "TensorInfo") + .def_readwrite("name", &TensorInfo::name) + .def_readwrite("shape", &TensorInfo::shape) + .def_readwrite("dtype", &TensorInfo::dtype); + pybind11::class_(m, "Runtime") - .def(pybind11::init([](RuntimeOption& option) { - Runtime* runtime = new Runtime(); - runtime->Init(option); - return runtime; - })) + .def(pybind11::init()) + .def("init", &Runtime::Init) .def("infer", [](Runtime& self, std::map& data) { std::vector inputs(data.size()); @@ -75,7 +78,12 @@ void BindRuntime(pybind11::module& m) { outputs[i].Numel() * FDDataTypeSize(outputs[i].dtype)); } return results; - }); + }) + .def("num_inputs", &Runtime::NumInputs) + .def("num_outputs", &Runtime::NumOutputs) + .def("get_input_info", &Runtime::GetInputInfo) + .def("get_output_info", &Runtime::GetOutputInfo) + .def_readonly("option", &Runtime::option); pybind11::enum_(m, "Backend", pybind11::arithmetic(), "Backend for inference.") @@ -103,11 +111,6 @@ void BindRuntime(pybind11::module& m) { .value("FP64", FDDataType::FP64) .value("UINT8", FDDataType::UINT8); - pybind11::class_(m, "TensorInfo") - .def_readwrite("name", &TensorInfo::name) - .def_readwrite("shape", &TensorInfo::shape) - .def_readwrite("dtype", &TensorInfo::dtype); - m.def("get_available_backends", []() { return GetAvailableBackends(); }); } diff --git a/setup.py b/setup.py index 12bfc4deb43..f0ff3f16dee 100644 --- a/setup.py +++ b/setup.py @@ -324,11 +324,18 @@ def run(self): shutil.copy("ThirdPartyNotices.txt", "fastdeploy") shutil.copy("LICENSE", "fastdeploy") depend_libs = list() + + # modify the search path of libraries + command = "patchelf --set-rpath '$ORIGIN/libs/' .setuptools-cmake-build/fastdeploy_main.cpython-36m-x86_64-linux-gnu.so" + # The sw_64 not suppot patchelf, so we just disable that. + if platform.machine() != 'sw_64' and platform.machine() != 'mips64': + assert os.system(command) == 0, "patch fastdeploy_main.cpython-36m-x86_64-linux-gnu.so failed, the command: {}".format(command) + for f in os.listdir(".setuptools-cmake-build"): if not os.path.isfile(os.path.join(".setuptools-cmake-build", f)): continue if f.count("libfastdeploy") > 0: - depend_libs.append(os.path.join(".setuptools-cmake-build", f)) + shutil.copy(os.path.join(".setuptools-cmake-build", f), "fastdeploy/libs") for dirname in os.listdir(".setuptools-cmake-build/third_libs/install"): for lib in os.listdir( os.path.join(".setuptools-cmake-build/third_libs/install",