diff --git a/tests/common/fixtures/advanced_reboot.py b/tests/common/fixtures/advanced_reboot.py index 03793711f92..09d76144f2d 100644 --- a/tests/common/fixtures/advanced_reboot.py +++ b/tests/common/fixtures/advanced_reboot.py @@ -91,6 +91,8 @@ def __init__(self, request, duthosts, duthost, ptfhost, localhost, tbinfo, creds self.allowMacJump = kwargs["allow_mac_jumping"] if "allow_mac_jumping" in kwargs else False self.advanceboot_loganalyzer = kwargs["advanceboot_loganalyzer"] if "advanceboot_loganalyzer"\ in kwargs else None + self.consistency_checker_provider = kwargs["consistency_checker_provider"] if "consistency_checker_provider"\ + in kwargs else None self.other_vendor_nos = kwargs['other_vendor_nos'] if 'other_vendor_nos' in kwargs else False self.__dict__.update(kwargs) self.__extractTestParam() @@ -557,6 +559,50 @@ def acl_manager_checker(self, error_list): if int(acl_proc_count) != 1: error_list.append("Expected one ACL manager process running. Actual: {}".format(acl_proc_count)) + def check_asic_and_db_consistency(self): + """ + Check ASIC_DB and ASIC consistency, logging out any inconsistencies that are found. + """ + if not self.consistency_checker_provider.is_consistency_check_supported(self.duthost): + os_version = self.duthost.image_facts()["ansible_facts"]["ansible_image_facts"]["current"] + platform = self.duthost.facts['platform'] + logger.info((f"Consistency check is not supported on this platform ({platform}) and " + f"version ({os_version})")) + return + + with self.consistency_checker_provider.get_consistency_checker(self.duthost) as consistency_checker: + inconsistencies = consistency_checker.check_consistency() + not_implemented_attributes = set() + mismatched_attributes = {} + failed_to_query_asic_attributes = {} + + for sai_object, summary in inconsistencies.items(): + # Not implemented attributes + object_name = sai_object.split(":")[1] + for attr in summary["attributeNotImplemented"]: + not_implemented_attributes.add(f"{object_name}.{attr}") + + # Mismatched attributes + mismatched_attributes = { + attr: summary["attributes"][attr] for attr + in summary["mismatchedAttributes"] + } + if mismatched_attributes: + mismatched_attributes[sai_object] = mismatched_attributes + + # Failed to query ASIC attributes + if summary["failedToQueryAsic"]: + failed_to_query_asic_attributes[sai_object] = summary["failedToQueryAsic"] + + if not_implemented_attributes: + logger.warning(f"Not implemented attributes: {not_implemented_attributes}") + + if mismatched_attributes: + logger.error(f"Mismatched attributes found: {mismatched_attributes}") + + if failed_to_query_asic_attributes: + logger.error(f"Failed to query ASIC attributes: {failed_to_query_asic_attributes}") + def runRebootTest(self): # Run advanced-reboot.ReloadTest for item in preboot/inboot list count = 0 @@ -597,6 +643,8 @@ def runRebootTest(self): finally: if self.postboot_setup: self.postboot_setup() + if self.consistency_checker_provider: + self.check_asic_and_db_consistency() # capture the test logs, and print all of them in case of failure, or a summary in case of success log_dir = self.__fetchTestLogs(rebootOper, log_dst_suffix=rebootOper) self.print_test_logs_summary(log_dir) @@ -677,6 +725,9 @@ def runMultiHopRebootTest(self, upgrade_path_urls, rebootOper=None, base_image_s self.duthost.hostname, upgrade_path_str)) if post_hop_teardown: post_hop_teardown(hop_index) + + if self.consistency_checker_provider: + self.check_asic_and_db_consistency() except Exception: traceback_msg = traceback.format_exc() err_msg = "Exception caught while running advanced-reboot test on ptf during upgrade {}: \n{}".format( diff --git a/tests/common/fixtures/consistency_checker/consistency_checker.py b/tests/common/fixtures/consistency_checker/consistency_checker.py index 44044027b2b..a1cdaf1ad9b 100644 --- a/tests/common/fixtures/consistency_checker/consistency_checker.py +++ b/tests/common/fixtures/consistency_checker/consistency_checker.py @@ -3,8 +3,10 @@ import json import os import datetime +from typing import List, Optional from collections import defaultdict -from tests.common.fixtures.consistency_checker.constants import SUPPORTED_PLATFORMS_AND_VERSIONS +from tests.common.fixtures.consistency_checker.constants import SUPPORTED_PLATFORMS_AND_VERSIONS, \ + ConsistencyCheckQueryKey, ALL_ATTRIBUTES logger = logging.getLogger(__name__) @@ -154,7 +156,7 @@ def get_db_and_asic_peers(self, keys=["*"]) -> dict: return dict(results) - def check_consistency(self, keys=["*"]) -> dict: + def check_consistency(self, keys=None) -> dict: """ Get the out-of-sync ASIC_DB and ASIC attributes. Differences are indicative of an error state. Same arg style as the get_objects function but returns a list of objects that don't match or couldn't @@ -163,7 +165,7 @@ def check_consistency(self, keys=["*"]) -> dict: :param keys: Optional list of glob search strings that correspond to the --key arg of sonic-db-dump. sonic-db-dump doesn't take multiple keys, so a list is passed in to support multiple - keys at the API level. + keys at the API level. If not provided, then the default keys are used. :return: Dictionary containing the out-of-sync ASIC_DB and ASIC attributes. Example return val (matching): @@ -186,11 +188,16 @@ def check_consistency(self, keys=["*"]) -> dict: "failedToQueryAsic": [ {"SAI_BUFFER_PROFILE_ATTR_SHARED_DYNAMIC_TH": "Failed to query attribute value"} ], - "mismatchedAttributes": ["SAI_BUFFER_PROFILE_ATTR_THRESHOLD_MODE"] + "mismatchedAttributes": ["SAI_BUFFER_PROFILE_ATTR_THRESHOLD_MODE"], + "attributeNotImplemented": ["SAI_BUFFER_PROFILE_ATTR_POOL_ID"] }, ... } """ + if keys is None: + platform = self._duthost.facts['platform'] + os_version = self._duthost.image_facts()["ansible_facts"]["ansible_image_facts"]["current"] + keys = self._get_consistency_checker_keys(platform, os_version) db_attributes = self._get_db_attributes(keys) asic_attributes = self._get_asic_attributes_from_db_results(db_attributes) @@ -198,7 +205,8 @@ def check_consistency(self, keys=["*"]) -> dict: inconsistencies = defaultdict(lambda: { "attributes": {}, "failedToQueryAsic": [], - "mismatchedAttributes": [] + "mismatchedAttributes": [], + "attributeNotImplemented": [], }) for object in db_attributes: @@ -231,23 +239,71 @@ def check_consistency(self, keys=["*"]) -> dict: if asic_query_success: inconsistencies[object]["mismatchedAttributes"].append(attr) else: - inconsistencies[object]["failedToQueryAsic"].append({attr: asic_object[attr]["error"]}) + error = asic_object[attr]["error"] + if "ATTR_NOT_IMPLEMENTED" in error: + inconsistencies[object]["attributeNotImplemented"].append(attr) + else: + inconsistencies[object]["failedToQueryAsic"].append({attr: error}) return dict(inconsistencies) - def _get_db_attributes(self, keys: list) -> dict: + def _get_consistency_checker_keys(self, platform, os_version) -> List[str]: + """ + Get the keys for the given platform and OS version. + + :param platform: Platform name + :param os_version: OS version + :return: List of keys + """ + + if platform not in SUPPORTED_PLATFORMS_AND_VERSIONS: + raise Exception(f"Unsupported platform: {platform}") + + supported_versions = SUPPORTED_PLATFORMS_AND_VERSIONS[platform] + for version in supported_versions: + if version in os_version: + return supported_versions[version] + + raise Exception(f"Unsupported OS version: {os_version}") + + def _get_db_attributes(self, keys: List[ConsistencyCheckQueryKey]) -> dict: """ Fetchs and merges the attributes of the objects returned by the search key from the DB. """ db_attributes = {} for key in keys: - result = self._duthost.command(f"sonic-db-dump -k '{key}' -n ASIC_DB") + result = self._duthost.command(f"sonic-db-dump -k '{key.key}' -n ASIC_DB") if result['rc'] != 0: raise Exception((f"Failed to fetch attributes for key '{key}' from ASIC_DB. " f"Return code: {result['rc']}, stdout: {result['stdout']}, " f"stderr: {result['stderr']}")) query_result = json.loads(result['stdout']) + + # Filter for attributes that we want ... + objects_with_no_attrs = [] + for object in query_result: + + if "NULL" in query_result[object]["value"]: + logger.debug(f"Ignoring attribute 'NULL' for object '{object}'") + del query_result[object]["value"]["NULL"] + + if ALL_ATTRIBUTES in key.attributes: + logger.debug(f"Retaining all attributes for object '{object}'") + else: + attributes_to_remove = set(query_result[object]["value"].keys()) - set(key.attributes) + for attr in attributes_to_remove: + logger.debug(f"Ignoring attribute '{attr}' for object '{object}'") + del query_result[object]["value"][attr] + + if len(query_result[object]["value"]) == 0: + objects_with_no_attrs.append(object) + + # ... then remove the objects that have no attributes left + for object in objects_with_no_attrs: + logger.debug(f"Ignoring empty object '{object}'") + del query_result[object] + db_attributes.update(query_result) return db_attributes @@ -304,6 +360,19 @@ def _get_asic_attributes_from_db_results(self, db_attributes: dict) -> dict: class ConsistencyCheckerProvider: + + def __init__(self, libsairedis_url_template: Optional[str], + python3_pysairedis_url_template: Optional[str]) -> None: + """ + The libsairedis_url_template and python3_pysairedis_url_template are optional URL templates that the + consistency checker can use to download the libsairedis and python3-pysairedis debs respectively. + + :param libsairedis_url_template: Optional URL template for the libsairedis deb + :param python3_pysairedis_url_template: Optional URL template for the python3-pysairedis deb + """ + self._libsairedis_url_template = libsairedis_url_template + self._python3_pysairedis_url_template = python3_pysairedis_url_template + def is_consistency_check_supported(self, dut) -> bool: """ Checks if the provided DUT is supported for consistency checking. @@ -318,29 +387,56 @@ def is_consistency_check_supported(self, dut) -> bool: current_version = dut.image_facts()['ansible_facts']['ansible_image_facts']['current'] supported_versions = SUPPORTED_PLATFORMS_AND_VERSIONS[platform] - if any(v in current_version for v in supported_versions): + if any(v in current_version for v in supported_versions.keys()): return True return False - def get_consistency_checker(self, dut, libsairedis_download_url=None, - python3_pysairedis_download_url=None) -> ConsistencyChecker: + def get_consistency_checker(self, dut) -> ConsistencyChecker: """ Get a new instance of the ConsistencyChecker class. :param dut: SonicHost object - :param libsairedis_download_url: Optional URL that the consistency checker should use to download the - libsairedis deb - :param python3_pysairedis_download_url: Optional URL that the consistency checker should use to - download the python3-pysairedis deb :return ConsistencyChecker: New instance of the ConsistencyChecker class """ + + os_version = dut.image_facts()["ansible_facts"]["ansible_image_facts"]["current"] + + if self._libsairedis_url_template or self._python3_pysairedis_url_template: + if "202305" in os_version: + sonic_version_template_param = "202305" + elif "202311" in os_version: + sonic_version_template_param = "202311" + else: + raise Exception(f"Unsupported OS version: {os_version}") + + libsairedis_download_url = self._libsairedis_url_template\ + .format(sonic_version=sonic_version_template_param)\ + if self._libsairedis_url_template else None + + python3_pysairedis_download_url = self._python3_pysairedis_url_template\ + .format(sonic_version=sonic_version_template_param)\ + if self._python3_pysairedis_url_template else None + return ConsistencyChecker(dut, libsairedis_download_url, python3_pysairedis_download_url) @pytest.fixture -def consistency_checker_provider(): +def consistency_checker_provider(request): """ Fixture that provides the ConsistencyCheckerProvider class. + + :param request: pytest request object """ - return ConsistencyCheckerProvider() + + if not request.config.getoption("enable_consistency_checker"): + logger.info("Consistency checker is not enabled. Skipping check.") + return None + + consistency_checker_libsairedis_url_template = request.config.getoption( + "consistency_checker_libsairedis_url_template") + consistency_checker_python3_pysairedis_url_template = request.config.getoption( + "consistency_checker_python3_pysairedis_url_template") + + return ConsistencyCheckerProvider(consistency_checker_libsairedis_url_template, + consistency_checker_python3_pysairedis_url_template) diff --git a/tests/common/fixtures/consistency_checker/constants.py b/tests/common/fixtures/consistency_checker/constants.py index f3a570fc060..ed0019bebe0 100644 --- a/tests/common/fixtures/consistency_checker/constants.py +++ b/tests/common/fixtures/consistency_checker/constants.py @@ -1,6 +1,51 @@ +from dataclasses import dataclass +from typing import List + +ALL_ATTRIBUTES = "all" + + +@dataclass +class ConsistencyCheckQueryKey: + key: str + attributes: List[str] + + +BROADCOM_KEYS: List[ConsistencyCheckQueryKey] = [ + ConsistencyCheckQueryKey("ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_POOL:*", attributes=[ALL_ATTRIBUTES]), + ConsistencyCheckQueryKey("ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_PROFILE:*", attributes=[ALL_ATTRIBUTES]), + ConsistencyCheckQueryKey("ASIC_STATE:SAI_OBJECT_TYPE_SWITCH:*", attributes=[ALL_ATTRIBUTES]), + ConsistencyCheckQueryKey("ASIC_STATE:SAI_OBJECT_TYPE_WRED:*", attributes=[ALL_ATTRIBUTES]), + ConsistencyCheckQueryKey( + "ASIC_STATE:SAI_OBJECT_TYPE_PORT:*", + attributes=[ + "SAI_PORT_ATTR_QOS_TC_TO_QUEUE_MAP", + "SAI_PORT_ATTR_QOS_TC_TO_PRIORITY_GROUP_MAP", + "SAI_PORT_ATTR_QOS_PFC_PRIORITY_TO_QUEUE_MAP", + "SAI_PORT_ATTR_QOS_DSCP_TO_TC_MAP", + "SAI_PORT_ATTR_MTU", + "SAI_PORT_ATTR_INGRESS_ACL", + "SAI_PORT_ATTR_AUTO_NEG_MODE", + "SAI_PORT_ATTR_PRIORITY_FLOW_CONTROL", + "SAI_PORT_ATTR_ADMIN_STATE", + "SAI_PORT_ATTR_FEC_MODE", + # The "get" implementation of the SAI_PORT_ATTR_SPEED attribute sometimes has a side effect of changing + # the port speed. Consistency-checker should not change the state of the DUT, so we ignore this attribute + # "SAI_PORT_ATTR_SPEED", + # This attribute doesn't match between ASIC_DB and ASIC SAI and the test fails the assertion + # "SAI_PORT_ATTR_PORT_VLAN_ID", + ] + ), +] + # The list of platforms and versions that have been tested to work with the consistency checker SUPPORTED_PLATFORMS_AND_VERSIONS = { - "x86_64-arista_7060_cx32s": ["202305", "202311"], - "x86_64-arista_7260cx3_64": ["202305", "202311"], + "x86_64-arista_7060_cx32s": { + "202305": BROADCOM_KEYS, + "202311": BROADCOM_KEYS, + }, + "x86_64-arista_7260cx3_64": { + "202305": BROADCOM_KEYS, + "202311": BROADCOM_KEYS, + }, } diff --git a/tests/common/helpers/upgrade_helpers.py b/tests/common/helpers/upgrade_helpers.py index 39073d7b3d3..25a713e7a2e 100644 --- a/tests/common/helpers/upgrade_helpers.py +++ b/tests/common/helpers/upgrade_helpers.py @@ -176,7 +176,8 @@ def upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, tbinfo, upgrade_type, get_advanced_reboot, advanceboot_loganalyzer, modify_reboot_script=None, allow_fail=False, sad_preboot_list=None, sad_inboot_list=None, reboot_count=1, - enable_cpa=False, preboot_setup=None, postboot_setup=None): + enable_cpa=False, preboot_setup=None, postboot_setup=None, + consistency_checker_provider=None): reboot_type = get_reboot_command(duthost, upgrade_type) if enable_cpa and "warm-reboot" in reboot_type: @@ -194,6 +195,7 @@ def upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, else: advancedReboot = get_advanced_reboot(rebootType=reboot_type, advanceboot_loganalyzer=advanceboot_loganalyzer, + consistency_checker_provider=consistency_checker_provider, allow_fail=allow_fail) for i in range(reboot_count): @@ -223,8 +225,9 @@ def upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, def multi_hop_warm_upgrade_test_helper(duthost, localhost, ptfhost, tbinfo, get_advanced_reboot, upgrade_type, upgrade_path_urls, base_image_setup=None, pre_hop_setup=None, - post_hop_teardown=None, multihop_advanceboot_loganalyzer_factory=None, - sad_preboot_list=None, sad_inboot_list=None, enable_cpa=False): + post_hop_teardown=None, consistency_checker_provider=None, + multihop_advanceboot_loganalyzer_factory=None, sad_preboot_list=None, + sad_inboot_list=None, enable_cpa=False): reboot_type = get_reboot_command(duthost, upgrade_type) if enable_cpa and "warm-reboot" in reboot_type: @@ -233,7 +236,8 @@ def multi_hop_warm_upgrade_test_helper(duthost, localhost, ptfhost, tbinfo, get_ ptf_ip = ptfhost.host.options['inventory_manager'].get_host(ptfhost.hostname).vars['ansible_host'] reboot_type = reboot_type + " -c {}".format(ptf_ip) - advancedReboot = get_advanced_reboot(rebootType=reboot_type) + advancedReboot = get_advanced_reboot(rebootType=reboot_type, + consistency_checker_provider=consistency_checker_provider) advancedReboot.runMultiHopRebootTestcase( upgrade_path_urls, base_image_setup=base_image_setup, pre_hop_setup=pre_hop_setup, post_hop_teardown=post_hop_teardown, @@ -242,48 +246,3 @@ def multi_hop_warm_upgrade_test_helper(duthost, localhost, ptfhost, tbinfo, get_ if enable_cpa and "warm-reboot" in reboot_type: ptfhost.shell('supervisorctl stop ferret') - - -def check_asic_and_db_consistency(pytest_config, duthost, consistency_checker_provider): - if not pytest_config.getoption("enable_consistency_checker"): - logger.info("Consistency checker is not enabled. Skipping check.") - return - - os_version = duthost.image_facts()["ansible_facts"]["ansible_image_facts"]["current"] - if not consistency_checker_provider.is_consistency_check_supported(duthost): - logger.info((f"Consistency check is not supported on this platform ({duthost.facts['platform']}) and " - f"version ({os_version})")) - return - - consistency_checker_libsairedis_url_template = pytest_config.getoption( - "consistency_checker_libsairedis_url_template") - consistency_checker_python3_pysairedis_url_template = pytest_config.getoption( - "consistency_checker_python3_pysairedis_url_template") - - if consistency_checker_libsairedis_url_template or consistency_checker_python3_pysairedis_url_template: - if "202305" in os_version: - sonic_version_template_param = "202305" - elif "202311" in os_version: - sonic_version_template_param = "202311" - else: - raise Exception(f"Unsupported OS version: {os_version}") - - libsairedis_download_url = consistency_checker_libsairedis_url_template\ - .format(sonic_version=sonic_version_template_param)\ - if consistency_checker_libsairedis_url_template else None - - python3_pysairedis_download_url = consistency_checker_python3_pysairedis_url_template\ - .format(sonic_version=sonic_version_template_param)\ - if consistency_checker_python3_pysairedis_url_template else None - - with consistency_checker_provider.get_consistency_checker(duthost, libsairedis_download_url, - python3_pysairedis_download_url) as consistency_checker: - keys = [ - "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_POOL:*", - "ASIC_STATE:SAI_OBJECT_TYPE_BUFFER_PROFILE:*", - "ASIC_STATE:SAI_OBJECT_TYPE_PORT:*", - "ASIC_STATE:SAI_OBJECT_TYPE_SWITCH:*", - "ASIC_STATE:SAI_OBJECT_TYPE_WRED:*", - ] - inconsistencies = consistency_checker.check_consistency(keys) - logger.warning(f"Found ASIC_DB and ASIC inconsistencies: {inconsistencies}") diff --git a/tests/platform_tests/test_advanced_reboot.py b/tests/platform_tests/test_advanced_reboot.py index 52246c4405a..e14bfac741b 100644 --- a/tests/platform_tests/test_advanced_reboot.py +++ b/tests/platform_tests/test_advanced_reboot.py @@ -6,6 +6,7 @@ from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 from tests.common.fixtures.duthost_utils import backup_and_restore_config_db # noqa F401 from tests.common.fixtures.advanced_reboot import get_advanced_reboot # noqa F401 +from tests.common.fixtures.consistency_checker.consistency_checker import consistency_checker_provider # noqa F401 from tests.platform_tests.verify_dut_health import add_fail_step_to_reboot # noqa F401 from tests.common.platform.warmboot_sad_cases import get_sad_case_list, SAD_CASE_LIST from tests.common.platform.device_utils import advanceboot_loganalyzer, verify_dut_health, advanceboot_neighbor_restore # noqa F401 @@ -78,7 +79,7 @@ def pytest_generate_tests(metafunc): # Tetcases to verify normal reboot procedure ### def test_fast_reboot(request, get_advanced_reboot, verify_dut_health, # noqa F811 - advanceboot_loganalyzer, # noqa F811 + advanceboot_loganalyzer, consistency_checker_provider, # noqa F811 capture_interface_counters): ''' Fast reboot test case is run using advanced reboot test fixture @@ -87,7 +88,8 @@ def test_fast_reboot(request, get_advanced_reboot, verify_dut_health, @param get_advanced_reboot: advanced reboot test fixture ''' advancedReboot = get_advanced_reboot(rebootType='fast-reboot', - advanceboot_loganalyzer=advanceboot_loganalyzer) + advanceboot_loganalyzer=advanceboot_loganalyzer, + consistency_checker_provider=consistency_checker_provider) advancedReboot.runRebootTestcase() @@ -103,7 +105,8 @@ def test_fast_reboot_from_other_vendor(duthosts, rand_one_dut_hostname, request ''' duthost = duthosts[rand_one_dut_hostname] advancedReboot = get_advanced_reboot(rebootType='fast-reboot', other_vendor_nos=True, - advanceboot_loganalyzer=advanceboot_loganalyzer) + advanceboot_loganalyzer=advanceboot_loganalyzer, + consistency_checker_provider=consistency_checker_provider) # Before rebooting, we will flush all unnecessary databases, to mimic reboot from other vendor. flush_dbs(duthost) advancedReboot.runRebootTestcase() @@ -111,7 +114,7 @@ def test_fast_reboot_from_other_vendor(duthosts, rand_one_dut_hostname, request @pytest.mark.device_type('vs') def test_warm_reboot(request, testing_config, get_advanced_reboot, verify_dut_health, # noqa F811 - duthosts, advanceboot_loganalyzer, # noqa F811 + duthosts, advanceboot_loganalyzer, consistency_checker_provider, # noqa F811 capture_interface_counters, toggle_all_simulator_ports, enum_rand_one_per_hwsku_frontend_hostname, # noqa F811 toggle_simulator_port_to_upper_tor): # noqa F811 @@ -135,13 +138,13 @@ def test_warm_reboot(request, testing_config, get_advanced_reboot, verify_dut_he toggle_simulator_port_to_upper_tor(itfs) advancedReboot = get_advanced_reboot(rebootType='warm-reboot', - advanceboot_loganalyzer=advanceboot_loganalyzer # noqa F811 - ) + advanceboot_loganalyzer=advanceboot_loganalyzer, # noqa F811 + consistency_checker_provider=consistency_checker_provider) advancedReboot.runRebootTestcase() def test_warm_reboot_mac_jump(request, get_advanced_reboot, verify_dut_health, # noqa F811 - advanceboot_loganalyzer, # noqa F811 + advanceboot_loganalyzer, consistency_checker_provider, # noqa F811 capture_interface_counters): ''' Warm reboot testcase with one MAC address (00-06-07-08-09-0A) jumping from @@ -157,7 +160,8 @@ def test_warm_reboot_mac_jump(request, get_advanced_reboot, verify_dut_health, generated during warm reboot will cause META checker failure resulting to Orchagent crash. ''' advancedReboot = get_advanced_reboot(rebootType='warm-reboot', allow_mac_jumping=True, - advanceboot_loganalyzer=advanceboot_loganalyzer) + advanceboot_loganalyzer=advanceboot_loganalyzer, + consistency_checker_provider=consistency_checker_provider) advancedReboot.runRebootTestcase() @@ -165,8 +169,8 @@ def test_warm_reboot_mac_jump(request, get_advanced_reboot, verify_dut_health, @pytest.mark.device_type('vs') def test_warm_reboot_sad(duthosts, rand_one_dut_hostname, nbrhosts, fanouthosts, vmhost, tbinfo, get_advanced_reboot, verify_dut_health, advanceboot_loganalyzer, # noqa F811 - backup_and_restore_config_db, advanceboot_neighbor_restore, # noqa F811 - sad_case_type): + consistency_checker_provider, backup_and_restore_config_db, # noqa F811 + advanceboot_neighbor_restore, sad_case_type): # noqa F811 ''' Warm reboot with sad path @param get_advanced_reboot: Fixture located in advanced_reboot.py @@ -178,7 +182,8 @@ def test_warm_reboot_sad(duthosts, rand_one_dut_hostname, nbrhosts, fanouthosts, ''' duthost = duthosts[rand_one_dut_hostname] advancedReboot = get_advanced_reboot(rebootType='warm-reboot', - advanceboot_loganalyzer=advanceboot_loganalyzer) + advanceboot_loganalyzer=advanceboot_loganalyzer, + consistency_checker_provider=consistency_checker_provider) sad_preboot_list, sad_inboot_list = get_sad_case_list( duthost, nbrhosts, fanouthosts, vmhost, tbinfo, sad_case_type) diff --git a/tests/upgrade_path/test_multi_hop_upgrade_path.py b/tests/upgrade_path/test_multi_hop_upgrade_path.py index c3405252521..2c9ba089644 100644 --- a/tests/upgrade_path/test_multi_hop_upgrade_path.py +++ b/tests/upgrade_path/test_multi_hop_upgrade_path.py @@ -9,7 +9,7 @@ from tests.common.platform.device_utils import check_neighbors, \ multihop_advanceboot_loganalyzer_factory, verify_dut_health, advanceboot_neighbor_restore # noqa F401 from tests.common.helpers.upgrade_helpers import SYSTEM_STABILIZE_MAX_TIME, check_copp_config, check_reboot_cause, \ - check_services, install_sonic, multi_hop_warm_upgrade_test_helper, check_asic_and_db_consistency + check_services, install_sonic, multi_hop_warm_upgrade_test_helper from tests.common.fixtures.duthost_utils import backup_and_restore_config_db # noqa F401 from tests.upgrade_path.utilities import cleanup_prev_images, boot_into_base_image from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 @@ -81,7 +81,6 @@ def post_hop_teardown(hop_index): check_services(duthost) check_neighbors(duthost, tbinfo) check_copp_config(duthost) - check_asic_and_db_consistency(request.config, duthost, consistency_checker_provider) logger.info("Finished post hop teardown for hop {} image {}".format(hop_index, to_image)) multi_hop_warm_upgrade_test_helper( @@ -141,7 +140,6 @@ def post_hop_teardown(hop_index): check_services(duthost) check_neighbors(duthost, tbinfo) check_copp_config(duthost) - check_asic_and_db_consistency(request.config, duthost, consistency_checker_provider) logger.info("Finished post hop teardown for hop {} image {}".format(hop_index, to_image)) multi_hop_warm_upgrade_test_helper( diff --git a/tests/upgrade_path/test_upgrade_path.py b/tests/upgrade_path/test_upgrade_path.py index f0a0cb4638a..128b3f9d140 100644 --- a/tests/upgrade_path/test_upgrade_path.py +++ b/tests/upgrade_path/test_upgrade_path.py @@ -1,6 +1,6 @@ import pytest import logging -from tests.common.helpers.upgrade_helpers import install_sonic, upgrade_test_helper, check_asic_and_db_consistency +from tests.common.helpers.upgrade_helpers import install_sonic, upgrade_test_helper from tests.common.helpers.upgrade_helpers import restore_image # noqa F401 from tests.upgrade_path.utilities import cleanup_prev_images, boot_into_base_image from tests.common.fixtures.advanced_reboot import get_advanced_reboot # noqa F401 @@ -70,14 +70,11 @@ def upgrade_path_preboot_setup(): setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo, upgrade_type) - def upgrade_path_postboot_setup(): - check_asic_and_db_consistency(request.config, duthost, consistency_checker_provider) - upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, tbinfo, upgrade_type, get_advanced_reboot, advanceboot_loganalyzer=advanceboot_loganalyzer, preboot_setup=upgrade_path_preboot_setup, - postboot_setup=upgrade_path_postboot_setup, + consistency_checker_provider=consistency_checker_provider, enable_cpa=enable_cpa, reboot_count=2) @@ -95,14 +92,11 @@ def upgrade_path_preboot_setup(): setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo, upgrade_type) - def upgrade_path_postboot_setup(): - check_asic_and_db_consistency(request.config, duthost, consistency_checker_provider) - upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, tbinfo, upgrade_type, get_advanced_reboot, advanceboot_loganalyzer=advanceboot_loganalyzer, preboot_setup=upgrade_path_preboot_setup, - postboot_setup=upgrade_path_postboot_setup, + consistency_checker_provider=consistency_checker_provider, enable_cpa=enable_cpa) @@ -121,15 +115,12 @@ def upgrade_path_preboot_setup(): setup_upgrade_test(duthost, localhost, from_image, to_image, tbinfo, upgrade_type) - def upgrade_path_postboot_setup(): - check_asic_and_db_consistency(request.config, duthost, consistency_checker_provider) - sad_preboot_list, sad_inboot_list = get_sad_case_list( duthost, nbrhosts, fanouthosts, vmhost, tbinfo, sad_case_type) upgrade_test_helper(duthost, localhost, ptfhost, from_image, to_image, tbinfo, "warm", get_advanced_reboot, advanceboot_loganalyzer=advanceboot_loganalyzer, preboot_setup=upgrade_path_preboot_setup, - postboot_setup=upgrade_path_postboot_setup, + consistency_checker_provider=consistency_checker_provider, sad_preboot_list=sad_preboot_list, sad_inboot_list=sad_inboot_list, enable_cpa=enable_cpa)