diff --git a/tests/common/plugins/sanity_check/__init__.py b/tests/common/plugins/sanity_check/__init__.py index bb0ef76a24b..0b2ff7d3102 100644 --- a/tests/common/plugins/sanity_check/__init__.py +++ b/tests/common/plugins/sanity_check/__init__.py @@ -20,8 +20,20 @@ def pytest_sessionfinish(session, exitstatus): - if session.config.cache.get("sanity_check_failed", None): - session.config.cache.set("sanity_check_failed", None) + + pre_sanity_failed = session.config.cache.get("pre_sanity_check_failed", None) + post_sanity_failed = session.config.cache.get("post_sanity_check_failed", None) + + if pre_sanity_failed: + session.config.cache.set("pre_sanity_check_failed", None) + if post_sanity_failed: + session.config.cache.set("post_sanity_check_failed", None) + + if pre_sanity_failed and not post_sanity_failed: + session.exitstatus = constants.PRE_SANITY_CHECK_FAILED_RC + elif not pre_sanity_failed and post_sanity_failed: + session.exitstatus = constants.POST_SANITY_CHECK_FAILED_RC + elif pre_sanity_failed and post_sanity_failed: session.exitstatus = constants.SANITY_CHECK_FAILED_RC @@ -224,7 +236,7 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): failed_results = [result for result in check_results if result['failed']] if failed_results: if not allow_recover: - request.config.cache.set("sanity_check_failed", True) + request.config.cache.set("pre_sanity_check_failed", True) pt_assert(False, "!!!!!!!!!!!!!!!!Pre-test sanity check failed: !!!!!!!!!!!!!!!!\n{}" .format(json.dumps(failed_results, indent=4, default=fallback_serializer))) else: @@ -252,7 +264,7 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): start_macsec_service() startup_macsec() except Exception as e: - request.config.cache.set("sanity_check_failed", True) + request.config.cache.set("pre_sanity_check_failed", True) logger.error("Recovery of sanity check failed with exception: ") pt_assert( False, @@ -267,7 +279,7 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): new_failed_results = [result for result in new_check_results if result['failed']] if new_failed_results: - request.config.cache.set("sanity_check_failed", True) + request.config.cache.set("pre_sanity_check_failed", True) pt_assert(False, "!!!!!!!!!!!!!!!! Pre-test sanity check after recovery failed: !!!!!!!!!!!!!!!!\n{}" .format(json.dumps(new_failed_results, indent=4, default=fallback_serializer))) @@ -290,7 +302,7 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): post_failed_results = [result for result in post_check_results if result['failed']] if post_failed_results: - request.config.cache.set("sanity_check_failed", True) + request.config.cache.set("post_sanity_check_failed", True) pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}" .format(json.dumps(post_failed_results, indent=4, default=fallback_serializer))) diff --git a/tests/common/plugins/sanity_check/constants.py b/tests/common/plugins/sanity_check/constants.py index 4c7ed3a3fad..a497caee415 100644 --- a/tests/common/plugins/sanity_check/constants.py +++ b/tests/common/plugins/sanity_check/constants.py @@ -21,15 +21,52 @@ # Recover related definitions RECOVER_METHODS = { - "config_reload": {"cmd": "bash -c 'config reload -y &>/dev/null'", "reboot": False, "adaptive": False, 'recover_wait': 120}, - "config_reload_f": {"cmd": "bash -c 'config reload -f -y &>/dev/null'", "reboot": False, "adaptive": False, 'recover_wait': 120}, - "load_minigraph": {"cmd": "bash -c 'config load_minigraph -y &>/dev/null'", "reboot": False, "adaptive": False, 'recover_wait': 60}, - "reboot": {"cmd": "reboot", "reboot": True, "adaptive": False, 'recover_wait': 120}, - "warm_reboot": {"cmd": "warm-reboot", "reboot": True, "adaptive": False, 'recover_wait': 120}, - "fast_reboot": {"cmd": "fast_reboot", "reboot": True, "adaptive": False, 'recover_wait': 120}, - "adaptive": {"cmd": None, "reboot": False, "adaptive": True, 'recover_wait': 30}, + "config_reload": { + "cmd": "bash -c 'config reload -y &>/dev/null'", + "reboot": False, + "adaptive": False, + 'recover_wait': 120 + }, + "config_reload_f": { + "cmd": "bash -c 'config reload -f -y &>/dev/null'", + "reboot": False, + "adaptive": False, + 'recover_wait': 120 + }, + "load_minigraph": { + "cmd": "bash -c 'config load_minigraph -y &>/dev/null'", + "reboot": False, + "adaptive": False, + 'recover_wait': 60 + }, + "reboot": { + "cmd": "reboot", + "reboot": True, + "adaptive": False, + 'recover_wait': 120 + }, + "warm_reboot": { + "cmd": "warm-reboot", + "reboot": True, + "adaptive": False, + 'recover_wait': 120 + }, + "fast_reboot": { + "cmd": "fast_reboot", + "reboot": True, + "adaptive": False, + 'recover_wait': 120 + }, + "adaptive": { + "cmd": None, + "reboot": False, + "adaptive": True, + 'recover_wait': 30 + }, } # All supported recover methods STAGE_PRE_TEST = 'stage_pre_test' STAGE_POST_TEST = 'stage_post_test' -SANITY_CHECK_FAILED_RC = 10 +PRE_SANITY_CHECK_FAILED_RC = 10 +POST_SANITY_CHECK_FAILED_RC = 11 +SANITY_CHECK_FAILED_RC = 12 diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 75fb6f50efe..65118279538 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -329,7 +329,8 @@ function run_individual_tests() rm -f ${LOG_PATH}/${test_dir}/${test_name}.log fi else - if [ ${ret_code} -eq 10 ]; then # rc 10 means sanity check failed + # rc 10 means pre-test sanity check failed, rc 12 means boths pre-test and post-test sanity check failed + if [ ${ret_code} -eq 10 ] || [${ret_code} -eq 12 ]; then echo "=== Sanity check failed for $test_script. Skip rest of the scripts if there is any. ===" return ${ret_code} fi