Skip to content
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions paddle/scripts/paddle_build.bat
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,12 @@ setlocal enabledelayedexpansion
:: if %errorlevel% NEQ 0 exit /b 8
:: for /F %%# in ('cmd /C nvidia-smi -L ^|find "GPU" /C') do set CUDA_DEVICE_COUNT=%%#
set CUDA_DEVICE_COUNT=1
echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

是不是叫win_cmake.sh更合适些

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

好的,我修改一下

-DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DON_INFER=%ON_INFER% ^
-DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^
-DINFERENCE_DEMO_INSTALL_DIR=%INFERENCE_DEMO_INSTALL_DIR% -DWITH_STATIC_LIB=%WITH_STATIC_LIB% ^
-DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^
-DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% >> %work_dir%\win_cmake.sh

%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\windows\run_unittests.sh %NIGHTLY_MODE% %PRECISION_TEST%

Expand Down
50 changes: 42 additions & 8 deletions tools/check_added_ut.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,32 +16,66 @@

set +e
set -x
SYSTEM=`uname -s`
if [ -z ${BRANCH} ]; then
BRANCH="develop"
fi

export CI_SKIP_CPP_TEST=OFF
PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../" && pwd )"
if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../" && pwd )"
elif [[ "$SYSTEM" == "Windows_NT" ]];then
PADDLE_ROOT="$(cd "$PWD/../" && pwd )"
fi
CURDIR=`pwd`
cd $PADDLE_ROOT
cp $PADDLE_ROOT/paddle/scripts/paddle_build.sh $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
cp $PADDLE_ROOT/paddle/scripts/paddle_build.sh $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
elif [[ "$SYSTEM" == "Windows_NT" ]];then
git remote | grep upstream
if [ $? != 0 ]; then
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
git fetch upstream develop
fi
fi
CURBRANCH=`git rev-parse --abbrev-ref HEAD`
echo $CURBRANCH
if [ `git branch | grep 'prec_added_ut'` ];then
git branch -D 'prec_added_ut'
fi
git checkout -b prec_added_ut upstream/${BRANCH}
git branch
mkdir prec_build
cd prec_build
bash $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh cmake_gen_in_current_dir >prebuild.log 2>&1
ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' > /$PADDLE_ROOT/br-ut
if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
bash $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh cmake_gen_in_current_dir >prebuild.log 2>&1
elif [[ "$SYSTEM" == "Windows_NT" ]];then
bash $PADDLE_ROOT/win_cmake.sh
fi
ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | grep 'test' > $PADDLE_ROOT/br-ut
cd $PADDLE_ROOT/build
ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' > /$PADDLE_ROOT/pr-ut
ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | grep 'test' > $PADDLE_ROOT/pr-ut
cd $PADDLE_ROOT
echo "================================="
echo "br-ut"
cat $PADDLE_ROOT/br-ut
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

br-ut的作用是

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

br-ut是为了获取upstream/develop分支上的单测,br-ut和pr-ut对比可以得到新增单测的名称

echo "================================="
echo "pr-ut"
cat $PADDLE_ROOT/pr-ut
echo "================================="
grep -F -x -v -f br-ut pr-ut > $PADDLE_ROOT/added_ut
sort pr-ut |uniq -d > $PADDLE_ROOT/duplicate_ut
if [[ "$SYSTEM" == 'Linux' ]];then
sort pr-ut |uniq -d > $PADDLE_ROOT/duplicate_ut
fi
echo "New-UT:"
cat $PADDLE_ROOT/added_ut
rm -rf prec_build
rm $PADDLE_ROOT/br-ut $PADDLE_ROOT/pr-ut $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
git checkout $CURBRANCH
if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
rm $PADDLE_ROOT/br-ut $PADDLE_ROOT/pr-ut $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
elif [[ "$SYSTEM" == "Windows_NT" ]];then
rm $PADDLE_ROOT/br-ut $PADDLE_ROOT/pr-ut $PADDLE_ROOT/get_added_ut.sh
fi
git checkout -f $CURBRANCH
echo $CURBRANCH
git branch -D prec_added_ut
cd $CURDIR
Expand Down
18 changes: 18 additions & 0 deletions tools/windows/run_unittests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ else
disable_ut_quickly=''
fi

# check added ut
set +e
cp $PADDLE_ROOT/tools/check_added_ut.sh $PADDLE_ROOT/tools/check_added_ut_win.sh
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个一定需要先cp吗 可以直接执行吗

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

直接执行,git会报冲突的错误,没有找到好的解决方法,只能先复制一个脚本执行,linux上也是复制了paddle_build.sh文件,然后执行cmake;
image

image

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK

bash $PADDLE_ROOT/tools/check_added_ut_win.sh
rm -rf $PADDLE_ROOT/tools/check_added_ut_win.sh
set -e


# /*==================Fixed Disabled Windows unittests==============================*/
# TODO: fix these unittest that is bound to fail
diable_wingpu_test="^lite_mul_model_test$|\
Expand Down Expand Up @@ -359,6 +367,16 @@ function show_ut_retry_result() {
}

set +e
if [ -f "$PADDLE_ROOT/added_ut" ];then
added_uts=^$(awk BEGIN{RS=EOF}'{gsub(/\n/,"$|^");print}' $PADDLE_ROOT/added_ut)$
ctest -R "(${added_uts})" --output-on-failure -C Release --repeat-until-fail 3;added_ut_error=$?
if [ "$added_ut_error" != 0 ];then
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

是不是在前面--timeout 150 设置了就可以,前面设的是150秒

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

好的,

echo "========================================"
echo "Added UT should not exceed 15 seconds"
Copy link
Contributor

@zhwesky2010 zhwesky2010 Mar 31, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个描述是不是不准确,因为看起来是新增单测失败了,不是超过15S挂了,然后上面还要加个TIMEOUT的新单测限定时间,因为是串行跑的,我建议和Linux限定一样为15S

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里的描述我改一下吧,加上时间限制的话,我觉得可以适当放宽一下,比如新增单测限制时间为30S,
Linux上的描述我一块改一下,增加"新增单测必须通过额外的三次执行检测"的描述;

echo "========================================"
exit 8;
fi
fi
run_unittest $eight_parallel_job 8
run_unittest $tetrad_parallel_jog 4
run_unittest $non_parallel_job_1
Expand Down