Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into npu_label_smooth_2

Author: Ray2020BD

.pre-commit-config.yaml

@@ -49,4 +49,7 @@ repos:
         entry: python ./tools/codestyle/copyright.hook
         language: system
         files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|py|sh)$
-        exclude: (?!.*third_party)^.*$ | (?!.*book)^.*$
+        exclude: |
+            (?x)^(
+                paddle/utils/.*
+            )$

CMakeLists.txt

@@ -222,6 +222,8 @@ option(WITH_MIPS   "Compile PaddlePaddle with mips support"         OFF)
 option(WITH_MUSL        "Compile with musl libc instead of gblic"  OFF)
 option(WITH_UNITY_BUILD "Compile with UnityBuild mode"             OFF)
 option(WITH_STRIP       "Strip so files of Whl packages"         OFF)
+option(NEW_RELEASE_CUBIN   "PaddlePaddle next-level release strategy for pypi cubin package"             OFF)
+option(NEW_RELEASE_JIT   "PaddlePaddle next-level release strategy for backup jit package"             OFF)
 
 # PY_VERSION
 if(NOT PY_VERSION)

cmake/configure.cmake

@@ -20,6 +20,10 @@ if(WITH_TESTING)
     add_definitions(-DPADDLE_WITH_TESTING)
 endif(WITH_TESTING)
 
+if(WITH_INFERENCE_API_TEST)
+    add_definitions(-DPADDLE_WITH_INFERENCE_API_TEST)
+endif(WITH_INFERENCE_API_TEST)
+
 if(NOT WITH_PROFILER)
     add_definitions(-DPADDLE_DISABLE_PROFILER)
 endif(NOT WITH_PROFILER)

cmake/cuda.cmake

@@ -3,10 +3,22 @@ if(NOT WITH_GPU)
 endif()
 
 
-if (WITH_NV_JETSON)
+if(WITH_NV_JETSON)
   add_definitions(-DWITH_NV_JETSON)
   set(paddle_known_gpu_archs "53 62 72")
   set(paddle_known_gpu_archs10 "53 62 72")
+elseif(NEW_RELEASE_CUBIN)
+  message("Using New Release Strategy - Cubin Packge")
+  add_definitions(-DNEW_RELEASE_CUBIN)
+  set(paddle_known_gpu_archs "35 37 50 52 60 61 70 75 80 86")
+  set(paddle_known_gpu_archs10 "50 60 70 75")
+  set(paddle_known_gpu_archs11 "60 70 75 80")
+elseif(NEW_RELEASE_JIT)
+  message("Using New Release Strategy - JIT Packge")
+  add_definitions(-DNEW_RELEASE_JIT)
+  set(paddle_known_gpu_archs "35 37 50 52 60 61 70 75 80 86")
+  set(paddle_known_gpu_archs10 "35 50 60 70 75")
+  set(paddle_known_gpu_archs11 "35 50 60 70 75 80")
 else()
   set(paddle_known_gpu_archs "35 50 52 60 61 70 75 80")
   set(paddle_known_gpu_archs10 "35 50 52 60 61 70 75")
@@ -130,11 +142,17 @@ function(select_nvcc_arch_flags out_variable)
     set(cuda_arch_bin ${CUDA_ARCH_BIN})
   endif()
 
+  if(NEW_RELEASE_JIT)
+      set(cuda_arch_ptx "${cuda_arch_ptx}${cuda_arch_bin}")
+      set(cuda_arch_bin "")
+  endif()
+
   # remove dots and convert to lists
   string(REGEX REPLACE "\\." "" cuda_arch_bin "${cuda_arch_bin}")
   string(REGEX REPLACE "\\." "" cuda_arch_ptx "${CUDA_ARCH_PTX}")
   string(REGEX MATCHALL "[0-9()]+" cuda_arch_bin "${cuda_arch_bin}")
   string(REGEX MATCHALL "[0-9]+"   cuda_arch_ptx "${cuda_arch_ptx}")
+
   list(REMOVE_DUPLICATES cuda_arch_bin)
   list(REMOVE_DUPLICATES cuda_arch_ptx)
 

cmake/cupti.cmake

@@ -9,6 +9,7 @@ find_path(CUPTI_INCLUDE_DIR cupti.h
         $ENV{CUPTI_ROOT} $ENV{CUPTI_ROOT}/include
         ${CUDA_TOOLKIT_ROOT_DIR}/extras/CUPTI/include
         ${CUDA_TOOLKIT_ROOT_DIR}/targets/x86_64-linux/include
+        ${CUDA_TOOLKIT_ROOT_DIR}/targets/aarch64-linux/include
         NO_DEFAULT_PATH
         )
 

cmake/external/mkldnn.cmake

@@ -20,7 +20,7 @@ SET(MKLDNN_SOURCE_DIR     ${THIRD_PARTY_PATH}/mkldnn/src/extern_mkldnn)
 SET(MKLDNN_INSTALL_DIR    ${THIRD_PARTY_PATH}/install/mkldnn)
 SET(MKLDNN_INC_DIR        "${MKLDNN_INSTALL_DIR}/include" CACHE PATH "mkldnn include directory." FORCE)
 SET(MKLDNN_REPOSITORY     ${GIT_URL}/oneapi-src/oneDNN.git)
-SET(MKLDNN_TAG            593e0de6267d2575f3e4c9e9818f0f11253d093a)
+SET(MKLDNN_TAG            e2d45252ae9c3e91671339579e3c0f0061f81d49)
 
 
 # Introduce variables:

cmake/external/python.cmake

@@ -27,7 +27,7 @@ print(s.get_config_var('LDVERSION') or s.get_config_var('VERSION'));
             OUTPUT_VARIABLE _PYTHON_VALUES
             ERROR_VARIABLE _PYTHON_ERROR_VALUE)
 
-    if(NOT _PYTHON_SUCCESS MATCHES 0)
+    if(NOT _PYTHON_SUCCESS EQUAL 0)
         set(PYTHONLIBS_FOUND FALSE)
         return()
     endif()

cmake/external/xpu.cmake

@@ -35,7 +35,7 @@ ELSE ()
 ENDIF()
 
 SET(XPU_BASE_URL_WITHOUT_DATE "https://baidu-kunlun-product.cdn.bcebos.com/KL-SDK/klsdk-dev")
-SET(XPU_BASE_URL "${XPU_BASE_URL_WITHOUT_DATE}/20210804")
+SET(XPU_BASE_URL "${XPU_BASE_URL_WITHOUT_DATE}/20210826")
 SET(XPU_XRE_URL  "${XPU_BASE_URL}/${XPU_XRE_DIR_NAME}.tar.gz" CACHE STRING "" FORCE)
 SET(XPU_XDNN_URL "${XPU_BASE_URL}/${XPU_XDNN_DIR_NAME}.tar.gz" CACHE STRING "" FORCE)
 SET(XPU_XCCL_URL "${XPU_BASE_URL_WITHOUT_DATE}/20210623/${XPU_XCCL_DIR_NAME}.tar.gz" CACHE STRING "" FORCE)

cmake/generic.cmake

@@ -431,6 +431,8 @@ function(cc_test_run TARGET_NAME)
     if (APPLE)
         set_tests_properties(${TARGET_NAME} PROPERTIES TIMEOUT 20)
     endif()
+  elseif(WITH_TESTING AND NOT TEST ${TARGET_NAME})
+    add_test(NAME ${TARGET_NAME} COMMAND ${CMAKE_COMMAND} -E echo CI skip ${TARGET_NAME}.)
   endif()
 endfunction()
 
@@ -459,6 +461,8 @@ function(cc_test TARGET_NAME)
         COMMAND ${TARGET_NAME}
         ARGS ${cc_test_ARGS})
     endif()
+  elseif(WITH_TESTING AND NOT TEST ${TARGET_NAME})
+    add_test(NAME ${TARGET_NAME} COMMAND ${CMAKE_COMMAND} -E echo CI skip ${TARGET_NAME}.)
   endif()
 endfunction(cc_test)
 

cmake/inference_lib.cmake

@@ -72,6 +72,12 @@ function(copy_part_of_thrid_party TARGET DST)
             copy(${TARGET}
                     SRCS ${MKLML_LIB} ${MKLML_IOMP_LIB} ${MKLML_INC_DIR}
                     DSTS ${dst_dir}/lib ${dst_dir}/lib ${dst_dir})
+            if(WITH_STRIP)
+                    add_custom_command(TARGET ${TARGET} POST_BUILD
+                            COMMAND strip -s ${dst_dir}/lib/libiomp5.so
+                            COMMAND strip -s ${dst_dir}/lib/libmklml_intel.so
+                            COMMENT "striping libiomp5.so\nstriping libmklml_intel.so")
+            endif()
         endif()
     elseif(${CBLAS_PROVIDER} STREQUAL EXTERN_OPENBLAS)
         set(dst_dir "${DST}/third_party/install/openblas")
@@ -94,8 +100,17 @@ function(copy_part_of_thrid_party TARGET DST)
                     DSTS ${dst_dir} ${dst_dir}/lib ${dst_dir}/lib)
         else()
             copy(${TARGET}
-                    SRCS ${MKLDNN_INC_DIR} ${MKLDNN_SHARED_LIB} ${MKLDNN_SHARED_LIB_1} ${MKLDNN_SHARED_LIB_2}
-                    DSTS ${dst_dir} ${dst_dir}/lib ${dst_dir}/lib ${dst_dir}/lib)
+                    SRCS ${MKLDNN_INC_DIR} ${MKLDNN_SHARED_LIB}
+                    DSTS ${dst_dir} ${dst_dir}/lib)
+            if(WITH_STRIP)
+                    add_custom_command(TARGET ${TARGET} POST_BUILD
+                            COMMAND strip -s ${dst_dir}/lib/libmkldnn.so.0
+                            COMMENT "striping libmkldnn.so.0")
+            endif()
+            add_custom_command(TARGET ${TARGET} POST_BUILD
+                    COMMAND ${CMAKE_COMMAND} -E create_symlink libmkldnn.so.0 ${dst_dir}/lib/libdnnl.so.1
+                    COMMAND ${CMAKE_COMMAND} -E create_symlink libmkldnn.so.0 ${dst_dir}/lib/libdnnl.so.2
+                    COMMENT "Make a symbol link of libmkldnn.so.0")
         endif()
     endif()
 
@@ -225,6 +240,13 @@ copy(inference_lib_dist
       SRCS  ${src_dir}/inference/capi_exp/pd_*.h  ${paddle_inference_c_lib}
       DSTS  ${PADDLE_INFERENCE_C_INSTALL_DIR}/paddle/include ${PADDLE_INFERENCE_C_INSTALL_DIR}/paddle/lib)
 
+if(WITH_STRIP AND NOT WIN32)
+        add_custom_command(TARGET inference_lib_dist POST_BUILD
+                COMMAND strip -s ${PADDLE_INFERENCE_C_INSTALL_DIR}/paddle/lib/libpaddle_inference_c.so
+                COMMAND strip -s ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/lib/libpaddle_inference.so
+                COMMENT "striping libpaddle_inference_c.so\nstriping libpaddle_inference.so")
+endif()
+
 # fluid library for both train and inference
 set(fluid_lib_deps inference_lib_dist)
 add_custom_target(fluid_lib_dist ALL DEPENDS ${fluid_lib_deps})