diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index a1b04cffbc3f93..5d1e5deb955e08 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -73,6 +73,7 @@ if not defined WITH_UNITY_BUILD set WITH_UNITY_BUILD=OFF if not defined NEW_RELEASE_ALL set NEW_RELEASE_ALL=ON if not defined NEW_RELEASE_PYPI set NEW_RELEASE_PYPI=OFF if not defined NEW_RELEASE_JIT set NEW_RELEASE_JIT=OFF +if not defined WITH_CPP_TEST set WITH_CPP_TEST=ON rem variable to control pipeline process if not defined WITH_TPCACHE set WITH_TPCACHE=OFF @@ -81,9 +82,15 @@ if not defined WITH_SCCACHE set WITH_SCCACHE=OFF if not defined INFERENCE_DEMO_INSTALL_DIR set INFERENCE_DEMO_INSTALL_DIR=%cache_dir:\=/%/inference_demo if not defined LOG_LEVEL set LOG_LEVEL=normal if not defined PRECISION_TEST set PRECISION_TEST=OFF +if not defined WIN_UNITTEST_LEVEL set WIN_UNITTEST_LEVEL=2 +rem LEVEL 0: For unittests unrelated to CUDA/TRT or unittests without GPU memory, only run on +rem PR-CI-Windows-Infernece(CUDA 11.2), skip them on PR-CI-Windows(CUDA 12.0) +rem LEVEL 1: For unittests unrelated to CUDA/TRT, only run on PR-CI-Windows-Infernece(CUDA 11.2), +rem skip them on PR-CI-Windows(CUDA 12.0) +rem LEVEL 2: run all test if not defined NIGHTLY_MODE set NIGHTLY_MODE=OFF if not defined retry_times set retry_times=1 -if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python37 +if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python38 if not defined BUILD_DIR set BUILD_DIR=build if not defined TEST_INFERENCE set TEST_INFERENCE=ON @@ -243,6 +250,7 @@ set MSVC_STATIC_CRT=OFF set ON_INFER=ON set WITH_TENSORRT=ON set WITH_INFERENCE_API_TEST=OFF +set WIN_UNITTEST_LEVEL=0 if not defined CUDA_ARCH_NAME set CUDA_ARCH_NAME=Auto call :cmake || goto cmake_error @@ -491,6 +499,12 @@ echo %task_name%|findstr build >nul && ( :cmake_impl cd /d %work_dir%\%BUILD_DIR% +rem whether to run cpp test +python -m pip install github +python -m pip install PyGithub +python %work_dir%\tools\windows\check_only_change_python_files.py +if exist %work_dir%\%BUILD_DIR%\only_change_python_file.txt set WITH_CPP_TEST=OFF +echo WITH_CPP_TEST: %WITH_CPP_TEST% echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^ -DWITH_INFERENCE_API_TEST=%WITH_INFERENCE_API_TEST% -DTHIRD_PARTY_PATH=%THIRD_PARTY_PATH% ^ @@ -498,7 +512,8 @@ echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -D -DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^ -DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% -DCUDA_ARCH_BIN=%CUDA_ARCH_BIN% -DCUB_PATH=%THIRD_PARTY_HOME%/cub ^ -DCUDA_TOOLKIT_ROOT_DIR="%CUDA_TOOLKIT_ROOT_DIR%" -DNEW_RELEASE_ALL=%NEW_RELEASE_ALL% -DNEW_RELEASE_PYPI=%NEW_RELEASE_PYPI% ^ --DNEW_RELEASE_JIT=%NEW_RELEASE_JIT% -DWITH_ONNXRUNTIME=%WITH_ONNXRUNTIME% +-DNEW_RELEASE_JIT=%NEW_RELEASE_JIT% -DWITH_ONNXRUNTIME=%WITH_ONNXRUNTIME% -DWITH_CPP_TEST=%WITH_CPP_TEST% ^ +-DWIN_UNITTEST_LEVEL=%WIN_UNITTEST_LEVEL% echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^ @@ -507,7 +522,8 @@ echo cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -D -DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^ -DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% -DCUDA_ARCH_BIN=%CUDA_ARCH_BIN% -DCUB_PATH=%THIRD_PARTY_HOME%/cub ^ -DCUDA_TOOLKIT_ROOT_DIR="%CUDA_TOOLKIT_ROOT_DIR%" -DNEW_RELEASE_ALL=%NEW_RELEASE_ALL% -DNEW_RELEASE_PYPI=%NEW_RELEASE_PYPI% ^ --DNEW_RELEASE_JIT=%NEW_RELEASE_JIT% -DWITH_ONNXRUNTIME=%WITH_ONNXRUNTIME% >> %work_dir%\win_cmake.sh +-DNEW_RELEASE_JIT=%NEW_RELEASE_JIT% -DWITH_ONNXRUNTIME=%WITH_ONNXRUNTIME% -DWITH_CPP_TEST=%WITH_CPP_TEST% ^ +-DWIN_UNITTEST_LEVEL=%WIN_UNITTEST_LEVEL% >> %work_dir%\win_cmake.sh cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_GPU=%WITH_GPU% -DWITH_MKL=%WITH_MKL% ^ -DWITH_TESTING=%WITH_TESTING% -DWITH_PYTHON=%WITH_PYTHON% -DPYTHON_EXECUTABLE=%PYTHON_EXECUTABLE% -DON_INFER=%ON_INFER% ^ @@ -516,7 +532,8 @@ cmake .. -G %GENERATOR% -DCMAKE_BUILD_TYPE=Release -DWITH_AVX=%WITH_AVX% -DWITH_ -DWITH_TENSORRT=%WITH_TENSORRT% -DTENSORRT_ROOT="%TENSORRT_ROOT%" -DMSVC_STATIC_CRT=%MSVC_STATIC_CRT% ^ -DWITH_UNITY_BUILD=%WITH_UNITY_BUILD% -DCUDA_ARCH_NAME=%CUDA_ARCH_NAME% -DCUDA_ARCH_BIN=%CUDA_ARCH_BIN% -DCUB_PATH=%THIRD_PARTY_HOME%/cub ^ -DCUDA_TOOLKIT_ROOT_DIR="%CUDA_TOOLKIT_ROOT_DIR%" -DNEW_RELEASE_ALL=%NEW_RELEASE_ALL% -DNEW_RELEASE_PYPI=%NEW_RELEASE_PYPI% ^ --DNEW_RELEASE_JIT=%NEW_RELEASE_JIT% -DWITH_ONNXRUNTIME=%WITH_ONNXRUNTIME% +-DNEW_RELEASE_JIT=%NEW_RELEASE_JIT% -DWITH_ONNXRUNTIME=%WITH_ONNXRUNTIME% -DWITH_CPP_TEST=%WITH_CPP_TEST% ^ +-DWIN_UNITTEST_LEVEL=%WIN_UNITTEST_LEVEL% goto:eof :cmake_error diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fd305ce6e89557..535d685a30a98c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -141,26 +141,48 @@ if(WITH_TESTING) add_subdirectory(ir/pir/cinn) endif() - add_subdirectory(amp) - add_subdirectory(asp) - add_subdirectory(autograd) + if(WIN32 AND WIN_UNITTEST_LEVEL LESS 2) + message(STATUS "Skip tests unrelated to CUDA/TRT") + else() + add_subdirectory(amp) + add_subdirectory(asp) + add_subdirectory(autograd) + add_subdirectory(custom_kernel) + add_subdirectory(custom_op) + add_subdirectory(custom_runtime) + add_subdirectory(dataset) + add_subdirectory(cpp_extension) + add_subdirectory(dygraph_to_static) + add_subdirectory(prim) + add_subdirectory(sot) + add_subdirectory(standalone_executor) + add_subdirectory(tokenizer) + add_subdirectory(rpc) + if(WITH_MKLDNN) + add_subdirectory(mkldnn) + endif() + endif() + add_subdirectory(book) # add_subdirectory(composite_ops) add_subdirectory(contrib) add_subdirectory(cpp) - add_subdirectory(custom_kernel) - add_subdirectory(custom_op) - add_subdirectory(custom_runtime) - add_subdirectory(dataset) - add_subdirectory(cpp_extension) + add_subdirectory(distribution) + add_subdirectory(ir) + add_subdirectory(indexing) + add_subdirectory(legacy_test) + add_subdirectory(quantization) + add_subdirectory(rnn) + add_subdirectory(sequence) + # add_subdirectory(white_list) + if(WITH_DISTRIBUTE) add_subdirectory(collective) add_subdirectory(auto_parallel) add_subdirectory(distributed_passes) add_subdirectory(ps) endif() - add_subdirectory(distribution) - add_subdirectory(dygraph_to_static) + if(NOT WIN32 OR NOT WITH_GPU) add_subdirectory(fft) endif() @@ -168,21 +190,7 @@ if(WITH_TESTING) if(WITH_IPU) add_subdirectory(ipu) endif() - add_subdirectory(ir) - add_subdirectory(indexing) - add_subdirectory(legacy_test) - if(WITH_MKLDNN) - add_subdirectory(mkldnn) - endif() - add_subdirectory(prim) - add_subdirectory(quantization) - add_subdirectory(rnn) - add_subdirectory(rpc) - add_subdirectory(sequence) - add_subdirectory(sot) - add_subdirectory(standalone_executor) - add_subdirectory(tokenizer) - # add_subdirectory(white_list) + if(WITH_XPU) add_subdirectory(xpu) endif() diff --git a/test/cpp/CMakeLists.txt b/test/cpp/CMakeLists.txt index 5256aec68452d9..80fa665640448a 100644 --- a/test/cpp/CMakeLists.txt +++ b/test/cpp/CMakeLists.txt @@ -1,3 +1,6 @@ +if(WIN32 AND NOT WITH_CPP_TEST) + return() +endif() add_subdirectory(auto_parallel) add_subdirectory(phi) add_subdirectory(jit) diff --git a/test/ir/CMakeLists.txt b/test/ir/CMakeLists.txt index 232ef033e2b35b..134783e11c35db 100644 --- a/test/ir/CMakeLists.txt +++ b/test/ir/CMakeLists.txt @@ -10,13 +10,16 @@ if(((NOT WITH_GPU) AND (NOT WITH_ROCM)) list(REMOVE_ITEM TEST_IR_PASSES test_ir_fusion_group_pass) endif() -foreach(target ${TEST_IR_PASSES}) - py_test_modules(${target} MODULES ${target}) - set_tests_properties(${target} PROPERTIES LABELS "RUN_TYPE=INFER") -endforeach() +if(WIN32 AND WIN_UNITTEST_LEVEL LESS 2) + message(STATUS "Skip tests unrelated to CUDA/TRT") +else() + foreach(target ${TEST_IR_PASSES}) + py_test_modules(${target} MODULES ${target}) + set_tests_properties(${target} PROPERTIES LABELS "RUN_TYPE=INFER") + endforeach() + add_subdirectory(pir) + set_tests_properties(test_fuse_resnet_unit PROPERTIES TIMEOUT 120) + set_tests_properties(test_convert_to_mixed_precision PROPERTIES TIMEOUT 300) +endif() add_subdirectory(inference) -add_subdirectory(pir) - -set_tests_properties(test_fuse_resnet_unit PROPERTIES TIMEOUT 120) -set_tests_properties(test_convert_to_mixed_precision PROPERTIES TIMEOUT 300) diff --git a/test/ir/inference/CMakeLists.txt b/test/ir/inference/CMakeLists.txt index 84abbaa986e61c..05dfc5c6fa53e1 100755 --- a/test/ir/inference/CMakeLists.txt +++ b/test/ir/inference/CMakeLists.txt @@ -49,8 +49,12 @@ if(WIN32) "test_trt_convert_quantize_dequantize_linear") list(REMOVE_ITEM TEST_TRT_IR_PASSES "test_trt_explicit_quantization") list(REMOVE_ITEM TEST_TRT_IR_PASSES "test_trt_explicit_quantization_resnet") + list(REMOVE_ITEM TEST_INFERENCE_IR_PASSES + "test_trt_explicit_quantization_resnet") list(REMOVE_ITEM TEST_TRT_IR_PASSES "test_trt_explicit_quantization_mobilenet") + list(REMOVE_ITEM TEST_INFERENCE_IR_PASSES + "test_trt_explicit_quantization_mobilenet") endif() # Only for cpu(mkl + openblas) @@ -110,7 +114,9 @@ foreach(TEST_INFERENCE_IR_PASS ${TEST_ONEDNN_IR_PASSES}) list(REMOVE_ITEM TEST_INFERENCE_IR_PASSES ${TEST_INFERENCE_IR_PASS}) endforeach() -if(WITH_MKLDNN) +if(WIN32 AND WIN_UNITTEST_LEVEL LESS 2) + message(STATUS "Skip tests unrelated to CUDA/TRT") +elseif(WITH_MKLDNN) foreach(target ${TEST_MKLDNN_IR_PASSES}) py_test_modules(${target} MODULES ${target}) set_tests_properties(${target} PROPERTIES LABELS "RUN_TYPE=INFER") @@ -175,9 +181,8 @@ if(WITH_GPU AND TENSORRT_FOUND) PROPERTIES TIMEOUT 300) set_tests_properties(test_trt_explicit_quantization_mobilenet PROPERTIES TIMEOUT 300) - endif() - if(WITH_MKLDNN) - set_tests_properties(test_save_optimized_model_pass PROPERTIES TIMEOUT 300) + else() + set_tests_properties(test_trt_convert_fill_constant PROPERTIES TIMEOUT 450) endif() if(WITH_NV_JETSON) @@ -208,9 +213,7 @@ if(WITH_GPU AND TENSORRT_FOUND) set_tests_properties(test_trt_nearest_interp_v2_op PROPERTIES TIMEOUT 30) set_tests_properties(test_trt_multiclass_nms3_op PROPERTIES TIMEOUT 60) - if(WITH_MKLDNN - AND TENSORRT_FOUND - AND WITH_GPU) + if(WITH_MKLDNN) set_tests_properties(test_merge_layernorm_fuse_pass PROPERTIES TIMEOUT 180) set_tests_properties(test_skip_merge_layernorm_fuse_pass PROPERTIES TIMEOUT 180) @@ -231,12 +234,6 @@ if(WITH_GPU AND TENSORRT_FOUND) PROPERTIES TIMEOUT 120) set_tests_properties(test_conv_elementwise_add_act_fuse_pass PROPERTIES TIMEOUT 120) - set_tests_properties(test_onednn_conv_bias_fuse_pass PROPERTIES TIMEOUT 300) - set_tests_properties(test_onednn_conv_concat_activation_fuse_pass - PROPERTIES TIMEOUT 300) - set_tests_properties(test_onednn_multi_gru_fuse_pass PROPERTIES TIMEOUT 120) - set_tests_properties(test_onednn_multi_gru_seq_fuse_pass PROPERTIES TIMEOUT - 120) set_tests_properties(test_flatten2_matmul_fuse_pass PROPERTIES TIMEOUT 240) set_tests_properties(test_squeeze2_matmul_fuse_pass PROPERTIES TIMEOUT 240) set_tests_properties(test_reshape2_matmul_fuse_pass PROPERTIES TIMEOUT 240) @@ -244,6 +241,12 @@ if(WITH_GPU AND TENSORRT_FOUND) 240) set_tests_properties(test_shuffle_channel_detect_pass PROPERTIES TIMEOUT 120) + set_tests_properties(test_conv_act_onednn_fuse_pass PROPERTIES TIMEOUT 120) + set_tests_properties(test_conv_transpose_eltwiseadd_bn_fuse_pass + PROPERTIES TIMEOUT 250) + set_tests_properties(test_conv_eltwiseadd_bn_fuse_pass PROPERTIES TIMEOUT + 300) + set_tests_properties(test_save_optimized_model_pass PROPERTIES TIMEOUT 300) if(WIN32) set_tests_properties(test_matmul_scale_fuse_pass PROPERTIES TIMEOUT 300) set_tests_properties(test_matmul_v2_scale_fuse_pass PROPERTIES TIMEOUT @@ -255,6 +258,16 @@ if(WITH_GPU AND TENSORRT_FOUND) set_tests_properties(test_map_matmul_to_mul_pass PROPERTIES TIMEOUT 360) set_tests_properties(test_layernorm_shift_partition_pass PROPERTIES TIMEOUT 360) + if(WIN_UNITTEST_LEVEL EQUAL 2) + set_tests_properties(test_onednn_conv_bias_fuse_pass PROPERTIES TIMEOUT + 300) + set_tests_properties(test_onednn_conv_concat_activation_fuse_pass + PROPERTIES TIMEOUT 300) + set_tests_properties(test_onednn_multi_gru_fuse_pass PROPERTIES TIMEOUT + 120) + set_tests_properties(test_onednn_multi_gru_seq_fuse_pass + PROPERTIES TIMEOUT 120) + endif() else() set_tests_properties(test_matmul_scale_fuse_pass PROPERTIES TIMEOUT 60) set_tests_properties(test_matmul_v2_scale_fuse_pass PROPERTIES TIMEOUT 60) @@ -272,41 +285,40 @@ if(WITH_GPU AND TENSORRT_FOUND) set_tests_properties(test_split_layernorm_to_math_ops_pass PROPERTIES TIMEOUT 240) endif() - endif() + if(WIN32 AND WIN_UNITTEST_LEVEL LESS 2) + message(STATUS "Skip tests unrelated to CUDA/TRT") + else() + set_tests_properties(test_onednn_conv_bn_fuse_pass PROPERTIES TIMEOUT 120) + set_tests_properties(test_onednn_conv_elementwise_add_fuse_pass + PROPERTIES TIMEOUT 120) + set_tests_properties(test_mkldnn_depthwise_conv_pass PROPERTIES TIMEOUT + 120) + set_tests_properties(test_onednn_reshape_transpose_matmul_fuse_pass + PROPERTIES TIMEOUT 100) + set_tests_properties(test_mkldnn_mish_op PROPERTIES TIMEOUT 300) + set_tests_properties(test_mkldnn_conv3d_op PROPERTIES TIMEOUT 300) + set_tests_properties(test_mkldnn_prelu_op PROPERTIES TIMEOUT 300) - if(WITH_MKLDNN) - set_tests_properties(test_onednn_conv_bn_fuse_pass PROPERTIES TIMEOUT 120) - set_tests_properties(test_onednn_conv_elementwise_add_fuse_pass - PROPERTIES TIMEOUT 120) - set_tests_properties(test_mkldnn_depthwise_conv_pass PROPERTIES TIMEOUT 120) - set_tests_properties(test_onednn_reshape_transpose_matmul_fuse_pass - PROPERTIES TIMEOUT 100) - set_tests_properties(test_mkldnn_mish_op PROPERTIES TIMEOUT 300) - set_tests_properties(test_mkldnn_conv3d_op PROPERTIES TIMEOUT 300) - set_tests_properties(test_mkldnn_prelu_op PROPERTIES TIMEOUT 300) - set_tests_properties(test_conv_act_onednn_fuse_pass PROPERTIES TIMEOUT 120) - set_tests_properties(test_conv_transpose_eltwiseadd_bn_fuse_pass - PROPERTIES TIMEOUT 250) - set_tests_properties(test_onednn_matmul_transpose_reshape_fuse_pass - PROPERTIES TIMEOUT 100) - set_tests_properties(test_conv_transpose_bn_fuse_pass PROPERTIES TIMEOUT - 300) - set_tests_properties(test_mkldnn_conv_hard_sigmoid_fuse_pass - PROPERTIES TIMEOUT 300) - set_tests_properties(test_mkldnn_conv_hard_swish_fuse_pass - PROPERTIES TIMEOUT 300) - set_tests_properties(test_onednn_batch_norm_act_fuse_pass PROPERTIES TIMEOUT - 100) - set_tests_properties(test_mkldnn_matmul_v2_transpose_reshape_fuse_pass - PROPERTIES TIMEOUT 100) - set_tests_properties(test_mkldnn_conv_transpose_bias_fuse_pass - PROPERTIES TIMEOUT 100) - set_tests_properties(test_conv_eltwiseadd_bn_fuse_pass PROPERTIES TIMEOUT + set_tests_properties(test_onednn_matmul_transpose_reshape_fuse_pass + PROPERTIES TIMEOUT 100) + set_tests_properties(test_conv_transpose_bn_fuse_pass PROPERTIES TIMEOUT + 300) + set_tests_properties(test_mkldnn_conv_hard_sigmoid_fuse_pass + PROPERTIES TIMEOUT 300) + set_tests_properties(test_mkldnn_conv_hard_swish_fuse_pass + PROPERTIES TIMEOUT 300) + set_tests_properties(test_onednn_batch_norm_act_fuse_pass + PROPERTIES TIMEOUT 100) + set_tests_properties(test_mkldnn_matmul_v2_transpose_reshape_fuse_pass + PROPERTIES TIMEOUT 100) + set_tests_properties(test_mkldnn_conv_transpose_bias_fuse_pass + PROPERTIES TIMEOUT 100) + set_tests_properties(test_mkldnn_conv_mish_fuse_pass PROPERTIES TIMEOUT 300) - set_tests_properties(test_mkldnn_conv_mish_fuse_pass PROPERTIES TIMEOUT 300) - set_tests_properties(test_onednn_fc_activation_fuse_pass PROPERTIES TIMEOUT - 300) - set_tests_properties(test_mkldnn_conv_affine_channel_fuse_pass - PROPERTIES TIMEOUT 60) + set_tests_properties(test_onednn_fc_activation_fuse_pass + PROPERTIES TIMEOUT 300) + set_tests_properties(test_mkldnn_conv_affine_channel_fuse_pass + PROPERTIES TIMEOUT 60) + endif() endif() endif() diff --git a/tools/group_case_for_parallel.py b/tools/group_case_for_parallel.py index 0f48c1db26918b..66187ca4b06078 100644 --- a/tools/group_case_for_parallel.py +++ b/tools/group_case_for_parallel.py @@ -29,9 +29,15 @@ def group_case_for_parallel(rootPath): 'exclusive_card_tests', 'exclusive_card_tests_mem0', ]: - os.system( - f'cd {rootPath}/tools && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test_bak_20230908/{filename} --no-check-certificate' - ) + OS_NAME = sys.platform + if OS_NAME.startswith('win'): + os.system( + f'cd {rootPath}/tools && wget --no-proxy https://paddle-windows.bj.bcebos.com/pre_test_bak_20230908/{filename} --no-check-certificate' + ) + else: + os.system( + f'cd {rootPath}/tools && wget --no-proxy https://paddle-docker-tar.bj.bcebos.com/pre_test_bak_20230908/{filename} --no-check-certificate' + ) # get nightly tests nightly_tests_file = open('%s/tools/nightly_case' % rootPath, 'r') diff --git a/tools/windows/check_only_change_python_files.py b/tools/windows/check_only_change_python_files.py new file mode 100644 index 00000000000000..98ee7ac3eaf01e --- /dev/null +++ b/tools/windows/check_only_change_python_files.py @@ -0,0 +1,74 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" For the PR that only modified the unit test, get cases in pull request. """ + +import os +import ssl +import sys + +from github import Github + +PADDLE_ROOT = os.getenv('PADDLE_ROOT', '/paddle/') +PADDLE_ROOT += '/' +PADDLE_ROOT = PADDLE_ROOT.replace('//', '/') +ssl._create_default_https_context = ssl._create_unverified_context + + +class PRChecker: + """PR Checker.""" + + def __init__(self): + self.github = Github(os.getenv('GITHUB_API_TOKEN'), timeout=60) + self.repo = self.github.get_repo('PaddlePaddle/Paddle') + self.pr = None + + def init(self): + """Get pull request.""" + pr_id = os.getenv('GIT_PR_ID') + if not pr_id: + print('PREC No PR ID') + sys.exit(0) + self.pr = self.repo.get_pull(int(pr_id)) + + def get_pr_files(self): + """Get files in pull request.""" + page = 0 + file_dict = {} + while True: + files = self.pr.get_files().get_page(page) + if not files: + break + for f in files: + file_dict[PADDLE_ROOT + f.filename] = f.status + page += 1 + print("pr modify files: %s" % file_dict) + return file_dict + + def check_only_change_python_file(self): + file_dict = self.get_pr_files() + for filename in file_dict: + if not ( + filename.startswith(PADDLE_ROOT + 'python/') + and filename.endswith('.py') + ): + return False + return True + + +if __name__ == '__main__': + pr_checker = PRChecker() + pr_checker.init() + if pr_checker.check_only_change_python_file(): + with open('only_change_python_file.txt', 'w') as f: + f.write('yes') diff --git a/tools/windows/run_unittests.sh b/tools/windows/run_unittests.sh index f99f7c8cc58e7a..e660bee55069b0 100644 --- a/tools/windows/run_unittests.sh +++ b/tools/windows/run_unittests.sh @@ -702,19 +702,23 @@ export FLAGS_call_stack_level=2 if [ "${WITH_GPU:-OFF}" == "ON" ];then single_ut_mem_0_startTime_s=`date +%s` - while read line - do - run_unittest_gpu "$line" 16 - done < $PADDLE_ROOT/tools/single_card_tests_mem0_new - single_ut_mem_0_endTime_s=`date +%s` - single_ut_mem_0_Time_s=`expr $single_ut_mem_0_endTime_s - $single_ut_mem_0_startTime_s` - echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: $single_ut_mem_0_Time_s s" + if [ ${WIN_UNITTEST_LEVEL:-2} == "0" ]; then + echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: 0 s" + else + while read line + do + run_unittest_gpu "$line" 16 + done < $PADDLE_ROOT/tools/single_card_tests_mem0_new + single_ut_mem_0_endTime_s=`date +%s` + single_ut_mem_0_Time_s=`expr $single_ut_mem_0_endTime_s - $single_ut_mem_0_startTime_s` + echo "ipipe_log_param_1_mem_0_TestCases_Total_Time: $single_ut_mem_0_Time_s s" + fi single_ut_startTime_s=`date +%s` while read line do num=`echo $line | awk -F"$" '{print NF-1}'` - para_num=`expr $num / 3` + para_num=`expr $num / 2` if [ $para_num -eq 0 ]; then para_num=4 fi @@ -737,7 +741,7 @@ if [ "${WITH_GPU:-OFF}" == "ON" ];then while read line do num=`echo $line | awk -F"$" '{print NF-1}'` - para_num=`expr $num / 3` + para_num=`expr $num / 2` if [ $para_num -eq 0 ]; then para_num=4 fi @@ -762,7 +766,7 @@ if [ "${WITH_GPU:-OFF}" == "ON" ];then while read line do num=`echo $line | awk -F"$" '{print NF-1}'` - para_num=`expr $num / 3` + para_num=`expr $num / 2` if [ $para_num -eq 0 ]; then para_num=4 fi @@ -775,7 +779,7 @@ if [ "${WITH_GPU:-OFF}" == "ON" ];then noparallel_ut_startTime_s=`date +%s` while read line do - run_unittest_gpu "$line" 3 + run_unittest_gpu "$line" 8 done < $PADDLE_ROOT/tools/no_parallel_case_file noparallel_ut_endTime_s=`date +%s` noparallel_ut_Time_s=`expr $noparallel_ut_endTime_s - $noparallel_ut_startTime_s`