Skip to content

【Hackathon 6th Fundable Projects 4 No.1】remove parallel executor unittest -part #63543

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
luotao1 merged 3 commits into from
Apr 17, 2024
Merged

【Hackathon 6th Fundable Projects 4 No.1】remove parallel executor unittest -part #63543

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
34d241f
remove parallel executor unittest
ccsuzzh Apr 15, 2024
be60897
Merge branch 'develop' into rm_pe_test
ccsuzzh Apr 15, 2024
c7b4919
fix bug
ccsuzzh Apr 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 1 addition & 82 deletions test/legacy_test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -444,29 +444,21 @@ list(REMOVE_ITEM TEST_OPS test_feed_data_check_shape_type)
list(REMOVE_ITEM TEST_OPS test_fetch_lod_tensor_array)
list(REMOVE_ITEM TEST_OPS test_warpctc_op)
list(REMOVE_ITEM TEST_OPS test_data_norm_op)
list(REMOVE_ITEM TEST_OPS test_parallel_executor_transformer)
list(REMOVE_ITEM TEST_OPS test_parallel_executor_transformer_auto_growth)
list(REMOVE_ITEM TEST_OPS test_bilinear_interp_op)
list(REMOVE_ITEM TEST_OPS test_nearest_interp_op)
list(REMOVE_ITEM TEST_OPS test_imperative_resnet)
list(REMOVE_ITEM TEST_OPS test_imperative_resnet_sorted_gradient)
list(REMOVE_ITEM TEST_OPS test_imperative_mnist_sorted_gradient)
list(REMOVE_ITEM TEST_OPS test_imperative_se_resnext)
list(REMOVE_ITEM TEST_OPS test_imperative_mnist)
list(REMOVE_ITEM TEST_OPS test_ir_memory_optimize_transformer)
list(REMOVE_ITEM TEST_OPS test_layers)
list(REMOVE_ITEM TEST_OPS test_parallel_executor_seresnext_base_cpu)
list(REMOVE_ITEM TEST_OPS test_parallel_executor_seresnext_with_reduce_cpu)
list(REMOVE_ITEM TEST_OPS
test_parallel_executor_seresnext_with_fuse_all_reduce_cpu)
list(REMOVE_ITEM TEST_OPS test_imperative_ocr_attention_model)
list(REMOVE_ITEM TEST_OPS test_async_ssa_graph_executor_mnist)
list(REMOVE_ITEM TEST_OPS test_install_check)
list(REMOVE_ITEM TEST_OPS test_basic_gru_api)
list(REMOVE_ITEM TEST_OPS test_basic_gru_unit_op)
list(REMOVE_ITEM TEST_OPS test_basic_lstm_api)
list(REMOVE_ITEM TEST_OPS test_basic_lstm_unit_op)
list(REMOVE_ITEM TEST_OPS test_fuse_all_reduce_pass)
list(REMOVE_ITEM TEST_OPS test_fuse_bn_act_pass)
list(REMOVE_ITEM TEST_OPS test_fuse_bn_add_act_pass)
list(REMOVE_ITEM TEST_OPS test_conv3d_transpose_op)
Expand Down Expand Up @@ -772,25 +764,12 @@ if(WITH_DISTRIBUTE)
endif()
endif()

py_test_modules(test_parallel_executor_transformer MODULES
test_parallel_executor_transformer)
if(WIN32)
py_test_modules(
test_parallel_executor_transformer_auto_growth MODULES
test_parallel_executor_transformer_auto_growth ENVS
FLAGS_allocator_strategy=auto_growth CUDA_VISIBLE_DEVICES=0)
py_test_modules(test_fuse_all_reduce_pass MODULES test_fuse_all_reduce_pass
ENVS CUDA_VISIBLE_DEVICES=0)
py_test_modules(test_feed_data_check_shape_type MODULES
test_feed_data_check_shape_type ENVS CUDA_VISIBLE_DEVICES=0)
py_test_modules(test_fetch_lod_tensor_array MODULES
test_fetch_lod_tensor_array ENVS CUDA_VISIBLE_DEVICES=0)
else()
py_test_modules(
test_parallel_executor_transformer_auto_growth MODULES
test_parallel_executor_transformer_auto_growth ENVS
FLAGS_allocator_strategy=auto_growth)
py_test_modules(test_fuse_all_reduce_pass MODULES test_fuse_all_reduce_pass)
py_test_modules(test_feed_data_check_shape_type MODULES
test_feed_data_check_shape_type)
py_test_modules(test_fetch_lod_tensor_array MODULES
Expand All @@ -815,38 +794,10 @@ py_test_modules(
FLAGS_cudnn_batchnorm_spatial_persistent=1
FLAGS_conv_workspace_size_limit=1000)

# NOTE: These unittests will appear NaN steadily in windows CI. After analysis,
# it is found that windows CI will run all the training unittests with the ON_INFER option turned on,
# which will not appear in other CIs. The calculation behavior of some ops in inference mode is
# inconsistent with that in non-inference mode.
if(WITH_PYTHON)
py_test_modules(test_parallel_executor_seresnext_base_cpu MODULES
test_parallel_executor_seresnext_base_cpu)
py_test_modules(test_parallel_executor_seresnext_with_reduce_cpu MODULES
test_parallel_executor_seresnext_with_reduce_cpu)
py_test_modules(
test_parallel_executor_seresnext_with_fuse_all_reduce_cpu MODULES
test_parallel_executor_seresnext_with_fuse_all_reduce_cpu)
set_tests_properties(test_parallel_executor_seresnext_base_cpu
PROPERTIES TIMEOUT 900)
set_tests_properties(test_parallel_executor_seresnext_base_cpu
PROPERTIES LABELS "RUN_TYPE=NIGHTLY")
set_tests_properties(test_parallel_executor_seresnext_with_reduce_cpu
PROPERTIES TIMEOUT 750)
set_tests_properties(test_parallel_executor_seresnext_with_reduce_cpu
PROPERTIES LABELS "RUN_TYPE=NIGHTLY")
set_tests_properties(test_parallel_executor_seresnext_with_fuse_all_reduce_cpu
PROPERTIES TIMEOUT 750)
set_tests_properties(test_parallel_executor_seresnext_with_fuse_all_reduce_cpu
PROPERTIES LABELS "RUN_TYPE=NIGHTLY")
endif()

if(NOT WIN32)
# TODO: fix these unittests failure on Windows
py_test_modules(test_layers MODULES test_layers ENVS
FLAGS_cudnn_deterministic=1)
py_test_modules(test_ir_memory_optimize_transformer MODULES
test_ir_memory_optimize_transformer)
endif()

if(WITH_HETERPS)
Expand All @@ -871,11 +822,7 @@ set_tests_properties(
test_data_norm_op test_dataloader_keep_order test_dataloader_unkeep_order
test_buffer_shared_memory_reuse_pass PROPERTIES LABELS "RUN_TYPE=DIST")
set_tests_properties(
test_sync_batch_norm_op
test_parallel_executor_seresnext_base_gpu
test_parallel_executor_seresnext_with_reduce_gpu
test_parallel_executor_seresnext_with_fuse_all_reduce_gpu
test_distributed_fused_lamb_op_with_clip
test_sync_batch_norm_op test_distributed_fused_lamb_op_with_clip
test_distributed_fused_lamb_op_without_clip
test_distributed_fused_lamb_op_with_gradient_merge
PROPERTIES LABELS "RUN_TYPE=DIST")
Expand Down Expand Up @@ -907,13 +854,6 @@ if(NOT WIN32)
set_tests_properties(test_multiprocess_reader_exception
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
set_tests_properties(test_layers PROPERTIES TIMEOUT 120)
if(WITH_NV_JETSON)
set_tests_properties(test_ir_memory_optimize_transformer PROPERTIES TIMEOUT
1200)
else()
set_tests_properties(test_ir_memory_optimize_transformer PROPERTIES TIMEOUT
120)
endif()
endif()

if(WITH_DISTRIBUTE)
Expand Down Expand Up @@ -950,7 +890,6 @@ set_tests_properties(test_cross_entropy_loss PROPERTIES TIMEOUT 180)
set_tests_properties(test_gru_unit_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_activation_nn_grad PROPERTIES TIMEOUT 250)
set_tests_properties(test_empty_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_transformer PROPERTIES TIMEOUT 120)
set_tests_properties(test_elementwise_div_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_regularizer_api PROPERTIES TIMEOUT 150)
set_tests_properties(test_multiclass_nms_op PROPERTIES TIMEOUT 120)
Expand All @@ -963,7 +902,6 @@ set_tests_properties(test_add_reader_dependency PROPERTIES TIMEOUT 120)
set_tests_properties(test_bilateral_slice_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_buffer_shared_memory_reuse_pass PROPERTIES TIMEOUT
120)
set_tests_properties(test_fuse_relu_depthwise_conv_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_fleet_util PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_transformer_sorted_gradient
PROPERTIES TIMEOUT 120)
Expand Down Expand Up @@ -1022,16 +960,12 @@ set_tests_properties(test_index_add_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_tensordot PROPERTIES TIMEOUT 200)
set_tests_properties(test_partial_eager_deletion_transformer PROPERTIES TIMEOUT
120)
set_tests_properties(test_parallel_executor_seresnext_with_reduce_gpu
PROPERTIES TIMEOUT 120)
set_tests_properties(test_dropout_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_argsort_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_gather_nd_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_nn_grad PROPERTIES TIMEOUT 180)
set_tests_properties(test_elementwise_sub_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_row_conv_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_parallel_executor_seresnext_with_fuse_all_reduce_gpu
PROPERTIES TIMEOUT 120)
set_tests_properties(test_distributed_fused_lamb_op_with_clip PROPERTIES TIMEOUT
240)
set_tests_properties(test_distributed_fused_lamb_op_without_clip
Expand All @@ -1041,8 +975,6 @@ set_tests_properties(test_distributed_fused_lamb_op_with_gradient_merge
set_tests_properties(test_elementwise_min_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_nan_inf PROPERTIES TIMEOUT 120)
set_tests_properties(test_deformable_conv_v1_op PROPERTIES TIMEOUT 300)
set_tests_properties(test_parallel_executor_transformer_auto_growth
PROPERTIES TIMEOUT 120)
set_tests_properties(test_elementwise_add_op PROPERTIES TIMEOUT 200)
if(NOT WITH_COVERAGE)
set_tests_properties(test_weight_decay PROPERTIES TIMEOUT 120)
Expand Down Expand Up @@ -1070,13 +1002,10 @@ set_tests_properties(test_imperative_optimizer_v2 PROPERTIES TIMEOUT 150)
set_tests_properties(test_partial_sum_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_cond PROPERTIES TIMEOUT 240)
set_tests_properties(test_sgd_op PROPERTIES TIMEOUT 250)
set_tests_properties(test_parallel_executor_seresnext_base_gpu
PROPERTIES TIMEOUT 120)
set_tests_properties(test_norm_nn_grad PROPERTIES TIMEOUT 180)
set_tests_properties(test_matrix_nms_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_generator_dataloader PROPERTIES TIMEOUT 120)
set_tests_properties(test_partial_concat_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_fuse_optimizer_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_softmax_with_cross_entropy_op PROPERTIES TIMEOUT 220)
set_tests_properties(test_reduce_op PROPERTIES TIMEOUT 500)
set_tests_properties(test_adam_optimizer_fp32_fp64 PROPERTIES TIMEOUT 120)
Expand All @@ -1102,7 +1031,6 @@ set_tests_properties(test_conv2d_op_depthwise_conv PROPERTIES TIMEOUT 120)
set_tests_properties(test_conv2d_api PROPERTIES TIMEOUT 120)
set_tests_properties(test_elementwise_mul_op PROPERTIES TIMEOUT 120)
set_tests_properties(test_cyclic_cifar_dataset PROPERTIES TIMEOUT 120)
set_tests_properties(test_fuse_all_reduce_pass PROPERTIES TIMEOUT 120)
set_tests_properties(test_dygraph_multi_forward PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_ocr_attention_model PROPERTIES TIMEOUT 120)
set_tests_properties(test_imperative_mnist PROPERTIES TIMEOUT 120)
Expand Down Expand Up @@ -1284,15 +1212,6 @@ foreach(TEST_CINN_OP ${TEST_CINN_OPS})
endforeach()

if(WITH_CINN AND WITH_TESTING)
set_tests_properties(
test_parallel_executor_run_cinn
PROPERTIES
LABELS
"RUN_TYPE=CINN"
ENVIRONMENT
FLAGS_allow_cinn_ops="conv2d;conv2d_grad;elementwise_add;elementwise_add_grad;relu;relu_grad;sum"
)

set_tests_properties(test_tile_op PROPERTIES TIMEOUT 300)
endif()

Expand Down
Loading