diff --git a/tools/xpu/check_xpu_dependence.sh b/tools/xpu/check_xpu_dependence.sh index 3091205f12ea3e..2dffe5fea6824b 100644 --- a/tools/xpu/check_xpu_dependence.sh +++ b/tools/xpu/check_xpu_dependence.sh @@ -23,6 +23,7 @@ fi xpu_base_url=$1 xccl_base_url=$2 +BOS_PATTERN="https://baidu-kunlun-product.su.bcebos.com" echo "xpu_base_url: $xpu_base_url" echo "xccl_base_url: $xccl_base_url" @@ -73,6 +74,18 @@ function check_files() { rm -rf ./$local_dir } +# check xpu_base_url type +if [[ $xpu_base_url != *"$BOS_PATTERN"* ]]; then + echo "The xpu_base_url does not contain bos url, assume it is local path" + if [[ ! -d $xpu_base_url ]]; then + echo "The xpu_base_url does not exist, please check it" + exit 1 + fi + exit 0 +else + echo "The URL is a bos url, will follow default download & compile logic" +fi + # XRE xre_tar_file_names=("xre-kylin_aarch64" "xre-bdcentos_x86_64" "xre-ubuntu_x86_64" "xre-centos7_x86_64") xre_inner_file_names=("include/xpu/runtime.h" "so/libxpurt.so") diff --git a/tools/xpu/pack_paddle_depence.sh b/tools/xpu/pack_paddle_depence.sh index 2df7c98cbf48ab..25fe0bd1c51b99 100644 --- a/tools/xpu/pack_paddle_depence.sh +++ b/tools/xpu/pack_paddle_depence.sh @@ -17,12 +17,6 @@ set -e set -x -if [[ $# -eq 8 ]]; then - echo "Compiling Paddle with XHPC" - XHPC_URL=$7 - XHPC_DIR_NAME=$8 -fi - XRE_URL=$1 XRE_DIR_NAME=$2 @@ -32,41 +26,108 @@ XDNN_DIR_NAME=$4 XCCL_URL=$5 XCCL_DIR_NAME=$6 -wget --no-check-certificate ${XRE_URL} -q -O xre.tar.gz -tar xvf xre.tar.gz - -wget --no-check-certificate ${XDNN_URL} -q -O xdnn.tar.gz -tar xvf xdnn.tar.gz +if [[ $# -eq 8 ]]; then + echo "Compiling Paddle with XHPC" + XHPC_URL=$7 + XHPC_DIR_NAME=$8 +elif [[ $# -eq 7 ]]; then + XHPC_DIR_NAME=$7 +fi -wget --no-check-certificate ${XCCL_URL} -q -O xccl.tar.gz -tar xvf xccl.tar.gz +BOS_PATTERN="https://baidu-kunlun-product.su.bcebos.com" +mkdir -p xpu/include/xhpc/xblas +mkdir -p xpu/include/xhpc/xfa mkdir -p xpu/include/xpu mkdir -p xpu/lib -if ! [ -z ${XHPC_URL} ]; then - echo "Compiling Paddle with XHPC" - echo "XHPC_URL: ${XHPC_URL}" - wget --no-check-certificate ${XHPC_URL} -q -O xhpc.tar.gz - tar xvf xhpc.tar.gz - - mkdir -p xpu/include/xhpc/xblas - mkdir -p xpu/include/xhpc/xfa - - cp -r ${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas - cp -r ${XHPC_DIR_NAME}/xblas/so/* xpu/lib/ - - cp -r ${XHPC_DIR_NAME}/xdnn/include/* xpu/include/ - cp -r ${XHPC_DIR_NAME}/xdnn/so/* xpu/lib - - cp -r ${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa - cp -r ${XHPC_DIR_NAME}/xfa/so/* xpu/lib/ +function download_from_bos() { + wget --no-check-certificate ${XRE_URL} -q -O xre.tar.gz + tar xvf xre.tar.gz + + wget --no-check-certificate ${XDNN_URL} -q -O xdnn.tar.gz + tar xvf xdnn.tar.gz + + wget --no-check-certificate ${XCCL_URL} -q -O xccl.tar.gz + tar xvf xccl.tar.gz +} + +function xhpc_prepare() { + if ! [ -z ${XHPC_URL} ]; then + echo "XHPC_URL: ${XHPC_URL}" + wget --no-check-certificate ${XHPC_URL} -q -O xhpc.tar.gz + tar xvf xhpc.tar.gz + + cp -r ${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas + cp -r ${XHPC_DIR_NAME}/xblas/so/* xpu/lib/ + + cp -r ${XHPC_DIR_NAME}/xdnn/include/* xpu/include/ + cp -r ${XHPC_DIR_NAME}/xdnn/so/* xpu/lib + + cp -r ${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa + cp -r ${XHPC_DIR_NAME}/xfa/so/* xpu/lib/ + else + cp -r ${XDNN_DIR_NAME}/include/xpu/* xpu/include/xpu/ + cp -r ${XDNN_DIR_NAME}/so/* xpu/lib/ + fi +} + +function local_prepare() { + # xre prepare + if [[ ! -d ${LOCAL_PATH}/${XRE_DIR_NAME} ]]; then + XRE_TAR_NAME=${XRE_DIR_NAME}.tar.gz + tar -zxf ${LOCAL_PATH}/${XRE_TAR_NAME} -C ${LOCAL_PATH} + fi + + # xccl prepare + if [[ ! -d ${LOCAL_PATH}/${XCCL_DIR_NAME} ]]; then + XCCL_TAR_NAME=${XCCL_DIR_NAME}.tar.gz + tar -zxf ${LOCAL_PATH}/${XCCL_TAR_NAME} -C ${LOCAL_PATH} + fi + + # xhpc prepare + if [[ ! -d ${LOCAL_PATH}/${XHPC_DIR_NAME} ]]; then + XHPC_TAR_NAME=${XHPC_DIR_NAME}.tar.gz + tar -zxf ${LOCAL_PATH}/${XHPC_TAR_NAME} -C ${LOCAL_PATH} + fi +} + +function local_assemble() { + # xre assemble + cp -r ${LOCAL_PATH}/$XRE_DIR_NAME/include/xpu/* xpu/include/xpu/ + cp -r ${LOCAL_PATH}/$XRE_DIR_NAME/so/libxpurt* xpu/lib/ + + # xccl assemble + cp -r ${LOCAL_PATH}/$XCCL_DIR_NAME/include/* xpu/include/xpu/ + cp -r ${LOCAL_PATH}/$XCCL_DIR_NAME/so/* xpu/lib/ + + # xhpc assemble + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xblas/include/* xpu/include/xhpc/xblas + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xblas/so/* xpu/lib/ + + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xdnn/include/* xpu/include/ + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xdnn/so/* xpu/lib + + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xfa/include/* xpu/include/xhpc/xfa + cp -r ${LOCAL_PATH}/${XHPC_DIR_NAME}/xfa/so/* xpu/lib/ +} + +if [[ $XRE_URL != *"$BOS_PATTERN"* ]]; then + # below is local way + build_from="local" + LOCAL_PATH=$(dirname "$XRE_URL") + echo "LOCAL_PATH: ${LOCAL_PATH}" + + local_prepare + local_assemble else - cp -r $XDNN_DIR_NAME/include/xpu/* xpu/include/xpu/ - cp -r $XDNN_DIR_NAME/so/* xpu/lib/ + # below is default way + build_from="bos" + download_from_bos + xhpc_prepare + + cp -r $XRE_DIR_NAME/include/xpu/* xpu/include/xpu/ + cp -r $XRE_DIR_NAME/so/libxpurt* xpu/lib/ + cp -r $XCCL_DIR_NAME/include/* xpu/include/xpu/ + cp -r $XCCL_DIR_NAME/so/* xpu/lib/ fi - -cp -r $XRE_DIR_NAME/include/xpu/* xpu/include/xpu/ -cp -r $XRE_DIR_NAME/so/libxpurt* xpu/lib/ -cp -r $XCCL_DIR_NAME/include/* xpu/include/xpu/ -cp -r $XCCL_DIR_NAME/so/* xpu/lib/