Skip to content

Support different RC in case of pre or post sanity check failed #6860

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wangxin merged 1 commit into from
Nov 24, 2022
Merged

Support different RC in case of pre or post sanity check failed #6860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions tests/common/plugins/sanity_check/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,20 @@


def pytest_sessionfinish(session, exitstatus):
if session.config.cache.get("sanity_check_failed", None):
session.config.cache.set("sanity_check_failed", None)

pre_sanity_failed = session.config.cache.get("pre_sanity_check_failed", None)
post_sanity_failed = session.config.cache.get("post_sanity_check_failed", None)

if pre_sanity_failed:
session.config.cache.set("pre_sanity_check_failed", None)
if post_sanity_failed:
session.config.cache.set("post_sanity_check_failed", None)

if pre_sanity_failed and not post_sanity_failed:
session.exitstatus = constants.PRE_SANITY_CHECK_FAILED_RC
elif not pre_sanity_failed and post_sanity_failed:
session.exitstatus = constants.POST_SANITY_CHECK_FAILED_RC
elif pre_sanity_failed and post_sanity_failed:
session.exitstatus = constants.SANITY_CHECK_FAILED_RC


Expand Down Expand Up @@ -224,7 +236,7 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo):
failed_results = [result for result in check_results if result['failed']]
if failed_results:
if not allow_recover:
request.config.cache.set("sanity_check_failed", True)
request.config.cache.set("pre_sanity_check_failed", True)
pt_assert(False, "!!!!!!!!!!!!!!!!Pre-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"
.format(json.dumps(failed_results, indent=4, default=fallback_serializer)))
else:
Expand Down Expand Up @@ -252,7 +264,7 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo):
start_macsec_service()
startup_macsec()
except Exception as e:
request.config.cache.set("sanity_check_failed", True)
request.config.cache.set("pre_sanity_check_failed", True)
logger.error("Recovery of sanity check failed with exception: ")
pt_assert(
False,
Expand All @@ -267,7 +279,7 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo):

new_failed_results = [result for result in new_check_results if result['failed']]
if new_failed_results:
request.config.cache.set("sanity_check_failed", True)
request.config.cache.set("pre_sanity_check_failed", True)
pt_assert(False,
"!!!!!!!!!!!!!!!! Pre-test sanity check after recovery failed: !!!!!!!!!!!!!!!!\n{}"
.format(json.dumps(new_failed_results, indent=4, default=fallback_serializer)))
Expand All @@ -290,7 +302,7 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo):

post_failed_results = [result for result in post_check_results if result['failed']]
if post_failed_results:
request.config.cache.set("sanity_check_failed", True)
request.config.cache.set("post_sanity_check_failed", True)
pt_assert(False, "!!!!!!!!!!!!!!!! Post-test sanity check failed: !!!!!!!!!!!!!!!!\n{}"
.format(json.dumps(post_failed_results, indent=4, default=fallback_serializer)))

Expand Down
53 changes: 45 additions & 8 deletions tests/common/plugins/sanity_check/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,52 @@

# Recover related definitions
RECOVER_METHODS = {
"config_reload": {"cmd": "bash -c 'config reload -y &>/dev/null'", "reboot": False, "adaptive": False, 'recover_wait': 120},
"config_reload_f": {"cmd": "bash -c 'config reload -f -y &>/dev/null'", "reboot": False, "adaptive": False, 'recover_wait': 120},
"load_minigraph": {"cmd": "bash -c 'config load_minigraph -y &>/dev/null'", "reboot": False, "adaptive": False, 'recover_wait': 60},
"reboot": {"cmd": "reboot", "reboot": True, "adaptive": False, 'recover_wait': 120},
"warm_reboot": {"cmd": "warm-reboot", "reboot": True, "adaptive": False, 'recover_wait': 120},
"fast_reboot": {"cmd": "fast_reboot", "reboot": True, "adaptive": False, 'recover_wait': 120},
"adaptive": {"cmd": None, "reboot": False, "adaptive": True, 'recover_wait': 30},
"config_reload": {
"cmd": "bash -c 'config reload -y &>/dev/null'",
"reboot": False,
"adaptive": False,
'recover_wait': 120
},
"config_reload_f": {
"cmd": "bash -c 'config reload -f -y &>/dev/null'",
"reboot": False,
"adaptive": False,
'recover_wait': 120
},
"load_minigraph": {
"cmd": "bash -c 'config load_minigraph -y &>/dev/null'",
"reboot": False,
"adaptive": False,
'recover_wait': 60
},
"reboot": {
"cmd": "reboot",
"reboot": True,
"adaptive": False,
'recover_wait': 120
},
"warm_reboot": {
"cmd": "warm-reboot",
"reboot": True,
"adaptive": False,
'recover_wait': 120
},
"fast_reboot": {
"cmd": "fast_reboot",
"reboot": True,
"adaptive": False,
'recover_wait': 120
},
"adaptive": {
"cmd": None,
"reboot": False,
"adaptive": True,
'recover_wait': 30
},
} # All supported recover methods

STAGE_PRE_TEST = 'stage_pre_test'
STAGE_POST_TEST = 'stage_post_test'
SANITY_CHECK_FAILED_RC = 10
PRE_SANITY_CHECK_FAILED_RC = 10
POST_SANITY_CHECK_FAILED_RC = 11
SANITY_CHECK_FAILED_RC = 12
3 changes: 2 additions & 1 deletion tests/run_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,8 @@ function run_individual_tests()
rm -f ${LOG_PATH}/${test_dir}/${test_name}.log
fi
else
if [ ${ret_code} -eq 10 ]; then # rc 10 means sanity check failed
# rc 10 means pre-test sanity check failed, rc 12 means boths pre-test and post-test sanity check failed
if [ ${ret_code} -eq 10 ] || [${ret_code} -eq 12 ]; then
echo "=== Sanity check failed for $test_script. Skip rest of the scripts if there is any. ==="
return ${ret_code}
fi
Expand Down