From 8980ade959b4a86d1446e11b7fbf9bf9a0d1e6cd Mon Sep 17 00:00:00 2001 From: xiaweijiang Date: Tue, 14 Mar 2023 02:54:09 +0000 Subject: [PATCH] Add wait time to verify mux status in sanity check in 202012 branch What is the motivation for this PR? After config reload in one tor of dual-tor, mux simulator will be inconsistent for several seconds, that's because mux simulator needs some time to toggle to correct status. How did you do it? Add wait time in sanity check to verify mux status, and check mux status later. --- tests/common/plugins/sanity_check/checks.py | 133 +++++++++++--------- 1 file changed, 72 insertions(+), 61 deletions(-) diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index 6b7debe9818..0c1284b92c6 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -23,8 +23,8 @@ 'check_bgp', 'check_dbmemory', 'check_monit', - 'check_mux_simulator', - 'check_secureboot'] + 'check_secureboot', + 'check_mux_simulator'] __all__ = CHECK_ITEMS @@ -452,69 +452,80 @@ def _check_single_intf_status(intf_status, expected_side): def _check_dut_mux_status(duthosts, duts_minigraph_facts): + def _verify_show_mux_status(): + duts_mux_status = duthosts.show_and_parse("show mux status") + + duts_parsed_mux_status.clear() + for dut_hostname, dut_mux_status in duts_mux_status.items(): + logger.info('Verify that "show mux status" has output ON {}'.format(dut_hostname)) + if len(dut_mux_status) == 0: + err_msg_from_mux_status.append('No mux status in output of "show mux status"') + return False + + logger.info('Verify that mux ports match vlan interfaces of DUT.') + vlan_intf_names = set() + for vlan in duts_minigraph_facts[dut_hostname]['minigraph_vlans'].values(): + vlan_intf_names = vlan_intf_names.union(set(vlan['members'])) + dut_mux_intfs = [] + for row in dut_mux_status: + dut_mux_intfs.append(row['port']) + if vlan_intf_names != set(dut_mux_intfs): + err_msg_from_mux_status.append('Mux ports mismatch vlan interfaces, please check output of "show mux status"') + return False + + logger.info('Verify mux status and parse active/standby side') + dut_parsed_mux_status = {} + for row in dut_mux_status: + # Verify that mux status is either active or standby + if row['status'] not in ['active', 'standby']: + err_msg_from_mux_status.append('Unexpected mux status "{}", please check output of "show mux status"'.format(row['status'])) + return False + + # Parse mux status, transform port name to port index, which is also mux index + port_name = row['port'] + port_idx = duts_minigraph_facts[dut_hostname]['minigraph_port_indices'][port_name] + + # Transform "active" and "standby" to active side which is "upper_tor" or "lower_tor" + status = row['status'] + if dut_hostname == dut_upper_tor.hostname: + # On upper tor, mux status "active" means that active side of mux is upper_tor + # mux status "standby" means that active side of mux is lower_tor + active_side = UPPER_TOR if status == 'active' else LOWER_TOR + else: + # On lower tor, mux status "active" means that active side of mux is lower_tor + # mux status "standby" means that active side of mux is upper_tor + active_side = UPPER_TOR if status == 'standby' else LOWER_TOR + dut_parsed_mux_status[str(port_idx)] = active_side + duts_parsed_mux_status[dut_hostname] = dut_parsed_mux_status + + logger.info('Verify that the mux status on both ToRs are consistent') + upper_tor_mux_status = duts_parsed_mux_status[dut_upper_tor.hostname] + lower_tor_mux_status = duts_parsed_mux_status[dut_lower_tor.hostname] + + logger.info('Verify that mux status is consistent on both ToRs.') + for port_idx in upper_tor_mux_status: + if upper_tor_mux_status[port_idx] != lower_tor_mux_status[port_idx]: + err_msg_from_mux_status.append('Inconsistent mux status on dualtors, please check output of "show mux status"') + return False + + logger.info('Check passed, return parsed mux status') + err_msg_from_mux_status.append("") + return True + dut_upper_tor = duthosts[0] dut_lower_tor = duthosts[1] - # Run "show mux status" on dualtor DUTs to collect mux status - duts_mux_status = duthosts.show_and_parse('show mux status') - - # Parse and basic check duts_parsed_mux_status = {} - for dut_hostname, dut_mux_status in duts_mux_status.items(): - - logger.info('Verify that "show mux status" has output ON {}'.format(dut_hostname)) - if len(dut_mux_status) == 0: - err_msg = 'No mux status in output of "show mux status"' - return False, err_msg, {} - - logger.info('Verify that mux ports match vlan interfaces of DUT.') - vlan_intf_names = set() - for vlan in duts_minigraph_facts[dut_hostname]['minigraph_vlans'].values(): - vlan_intf_names = vlan_intf_names.union(set(vlan['members'])) - dut_mux_intfs = [] - for row in dut_mux_status: - dut_mux_intfs.append(row['port']) - if vlan_intf_names != set(dut_mux_intfs): - err_msg = 'Mux ports mismatch vlan interfaces, please check output of "show mux status"' - return False, err_msg, {} - - logger.info('Verify mux status and parse active/standby side') - dut_parsed_mux_status = {} - for row in dut_mux_status: - # Verify that mux status is either active or standby - if row['status'] not in ['active', 'standby']: - err_msg = 'Unexpected mux status "{}", please check output of "show mux status"'.format(row['status']) - return False, err_msg, {} - - # Parse mux status, transform port name to port index, which is also mux index - port_name = row['port'] - port_idx = duts_minigraph_facts[dut_hostname]['minigraph_port_indices'][port_name] - - # Transform "active" and "standby" to active side which is "upper_tor" or "lower_tor" - status = row['status'] - if dut_hostname == dut_upper_tor.hostname: - # On upper tor, mux status "active" means that active side of mux is upper_tor - # mux status "standby" means that active side of mux is lower_tor - active_side = UPPER_TOR if status == 'active' else LOWER_TOR - else: - # On lower tor, mux status "active" means that active side of mux is lower_tor - # mux status "standby" means that active side of mux is upper_tor - active_side = UPPER_TOR if status == 'standby' else LOWER_TOR - dut_parsed_mux_status[str(port_idx)] = active_side - duts_parsed_mux_status[dut_hostname] = dut_parsed_mux_status - - logger.info('Verify that the mux status on both ToRs are consistent') - upper_tor_mux_status = duts_parsed_mux_status[dut_upper_tor.hostname] - lower_tor_mux_status = duts_parsed_mux_status[dut_lower_tor.hostname] - - logger.info('Verify that mux status is consistent on both ToRs.') - for port_idx in upper_tor_mux_status: - if upper_tor_mux_status[port_idx] != lower_tor_mux_status[port_idx]: - err_msg = 'Inconsistent mux status on dualtors, please check output of "show mux status"' - return False, err_msg, {} - - logger.info('Check passed, return parsed mux status') - return True, "", upper_tor_mux_status + err_msg_from_mux_status = [] + + if not wait_until(30, 5, 0, _verify_show_mux_status): + if err_msg_from_mux_status: + err_msg = err_msg_from_mux_status[-1] + else: + err_msg = "Unknown error occured inside the check" + return False, err_msg, {} + + return True, "", duts_parsed_mux_status @pytest.fixture(scope='module')