[XPU] add support for local path compilation

tools/xpu/check_xpu_dependence.sh

@@ -23,6 +23,7 @@ fi
 
 xpu_base_url=$1
 xccl_base_url=$2
+BOS_PATTERN="https://baidu-kunlun-product.su.bcebos.com"
 
 echo "xpu_base_url: $xpu_base_url"
 echo "xccl_base_url: $xccl_base_url"
@@ -73,6 +74,18 @@ function check_files() {
     rm -rf ./$local_dir
 }
 
+# check xpu_base_url type
+if [[ $xpu_base_url != *"$BOS_PATTERN"* ]]; then
+    echo "The xpu_base_url does not contain bos url, assume it is local path"
+    if [[ ! -d $xpu_base_url ]]; then
+        echo "The xpu_base_url does not exist, please check it"
+        exit 1
+    fi
+    exit 0
+else
+    echo "The URL is a bos url, will follow default download & compile logic"
+fi
+
 # XRE
 xre_tar_file_names=("xre-kylin_aarch64" "xre-bdcentos_x86_64" "xre-ubuntu_x86_64" "xre-centos7_x86_64")
 xre_inner_file_names=("include/xpu/runtime.h" "so/libxpurt.so")

tools/xpu/pack_paddle_depence.sh

@@ -17,12 +17,6 @@
 set -e
 set -x
 
-if [[ $# -eq 8 ]]; then
-  echo "Compiling Paddle with XHPC"
-  XHPC_URL=$7
-  XHPC_DIR_NAME=$8
-fi
-
 XRE_URL=$1
 XRE_DIR_NAME=$2
 
@@ -32,41 +26,108 @@ XDNN_DIR_NAME=$4
 XCCL_URL=$5
 XCCL_DIR_NAME=$6
 
-wget --no-check-certificate ${XRE_URL} -q -O xre.tar.gz
-tar xvf xre.tar.gz
-
-wget --no-check-certificate ${XDNN_URL} -q -O xdnn.tar.gz
-tar xvf xdnn.tar.gz
+if [[ $# -eq 8 ]]; then
+  echo "Compiling Paddle with XHPC"
+  XHPC_URL=$7
+  XHPC_DIR_NAME=$8
+elif [[ $# -eq 7 ]]; then
+  XHPC_DIR_NAME=$7
+fi
 
-wget --no-check-certificate ${XCCL_URL} -q -O xccl.tar.gz
-tar xvf xccl.tar.gz
+BOS_PATTERN="https://baidu-kunlun-product.su.bcebos.com"
 
+mkdir -p xpu/include/xhpc/xblas
+mkdir -p xpu/include/xhpc/xfa
 mkdir -p xpu/include/xpu
 mkdir -p xpu/lib
 
-if ! [ -z ${XHPC_URL} ]; then
-  echo "Compiling Paddle with XHPC"
-  echo "XHPC_URL: ${XHPC_URL}"
-  wget --no-check-certificate ${XHPC_URL} -q -O xhpc.tar.gz
-  tar xvf xhpc.tar.gz
-
-  mkdir -p xpu/include/xhpc/xblas
-  mkdir -p xpu/include/xhpc/xfa
-
-  cp -r ${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas
-  cp -r ${XHPC_DIR_NAME}/xblas/so/* xpu/lib/
-
-  cp -r ${XHPC_DIR_NAME}/xdnn/include/* xpu/include/
-  cp -r ${XHPC_DIR_NAME}/xdnn/so/* xpu/lib
-
-  cp -r ${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa
-  cp -r ${XHPC_DIR_NAME}/xfa/so/* xpu/lib/
+function download_from_bos() {
+  wget --no-check-certificate ${XRE_URL} -q -O xre.tar.gz
+  tar xvf xre.tar.gz
+
+  wget --no-check-certificate ${XDNN_URL} -q -O xdnn.tar.gz
+  tar xvf xdnn.tar.gz
+
+  wget --no-check-certificate ${XCCL_URL} -q -O xccl.tar.gz
+  tar xvf xccl.tar.gz
+}
+
+function xhpc_prepare() {
+    if ! [ -z ${XHPC_URL} ]; then
+      echo "XHPC_URL: ${XHPC_URL}"
+      wget --no-check-certificate ${XHPC_URL} -q -O xhpc.tar.gz
+      tar xvf xhpc.tar.gz
+
+      cp -r ${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas
+      cp -r ${XHPC_DIR_NAME}/xblas/so/* xpu/lib/
+
+      cp -r ${XHPC_DIR_NAME}/xdnn/include/* xpu/include/
+      cp -r ${XHPC_DIR_NAME}/xdnn/so/* xpu/lib
+
+      cp -r ${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa
+      cp -r ${XHPC_DIR_NAME}/xfa/so/* xpu/lib/
+    else
+      cp -r ${XDNN_DIR_NAME}/include/xpu/* xpu/include/xpu/
+      cp -r ${XDNN_DIR_NAME}/so/* xpu/lib/
+    fi
+}
+
+function local_prepare() {
+    # xre prepare
+    if [[ ! -d ${LOCAL_PATH}/${XRE_DIR_NAME} ]]; then
+        XRE_TAR_NAME=${XRE_DIR_NAME}.tar.gz
+        tar -zxf  ${LOCAL_PATH}/${XRE_TAR_NAME} -C ${LOCAL_PATH}
+    fi
+
+    # xccl prepare
+    if [[ ! -d ${LOCAL_PATH}/${XCCL_DIR_NAME} ]]; then
+        XCCL_TAR_NAME=${XCCL_DIR_NAME}.tar.gz
+        tar -zxf  ${LOCAL_PATH}/${XCCL_TAR_NAME} -C ${LOCAL_PATH}
+    fi
+
+    # xhpc prepare
+    if [[ ! -d ${LOCAL_PATH}/${XHPC_DIR_NAME} ]]; then
+        XHPC_TAR_NAME=${XHPC_DIR_NAME}.tar.gz
+        tar -zxf  ${LOCAL_PATH}/${XHPC_TAR_NAME} -C ${LOCAL_PATH}
+    fi
+}
+
+function local_assemble() {
+    # xre assemble
+    cp -r ${LOCAL_PATH}/$XRE_DIR_NAME/include/xpu/* xpu/include/xpu/
+    cp -r ${LOCAL_PATH}/$XRE_DIR_NAME/so/libxpurt* xpu/lib/
+
+    # xccl assemble
+    cp -r ${LOCAL_PATH}/$XCCL_DIR_NAME/include/* xpu/include/xpu/
+    cp -r ${LOCAL_PATH}/$XCCL_DIR_NAME/so/* xpu/lib/
+
+    # xhpc assemble
+    cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas
+    cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xblas/so/* xpu/lib/
+
+    cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xdnn/include/* xpu/include/
+    cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xdnn/so/* xpu/lib
+
+    cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa
+    cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xfa/so/* xpu/lib/
+}
+
+if [[ $XRE_URL != *"$BOS_PATTERN"* ]]; then
+    # below is local way
+    build_from="local"
+    LOCAL_PATH=$(dirname "$XRE_URL")
+    echo "LOCAL_PATH: ${LOCAL_PATH}"
+
+    local_prepare
+    local_assemble
 else
-  cp -r $XDNN_DIR_NAME/include/xpu/* xpu/include/xpu/
-  cp -r $XDNN_DIR_NAME/so/* xpu/lib/
+    # below is default way
+    build_from="bos"
+    download_from_bos
+    xhpc_prepare
+
+    cp -r $XRE_DIR_NAME/include/xpu/* xpu/include/xpu/
+    cp -r $XRE_DIR_NAME/so/libxpurt* xpu/lib/
+    cp -r $XCCL_DIR_NAME/include/* xpu/include/xpu/
+    cp -r $XCCL_DIR_NAME/so/* xpu/lib/
 fi
-
-cp -r $XRE_DIR_NAME/include/xpu/* xpu/include/xpu/
-cp -r $XRE_DIR_NAME/so/libxpurt* xpu/lib/
-cp -r $XCCL_DIR_NAME/include/* xpu/include/xpu/
-cp -r $XCCL_DIR_NAME/so/* xpu/lib/