diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index 6b7debe9818..0c1284b92c6 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -23,8 +23,8 @@ 'check_bgp', 'check_dbmemory', 'check_monit', - 'check_mux_simulator', - 'check_secureboot'] + 'check_secureboot', + 'check_mux_simulator'] __all__ = CHECK_ITEMS @@ -452,69 +452,80 @@ def _check_single_intf_status(intf_status, expected_side): def _check_dut_mux_status(duthosts, duts_minigraph_facts): + def _verify_show_mux_status(): + duts_mux_status = duthosts.show_and_parse("show mux status") + + duts_parsed_mux_status.clear() + for dut_hostname, dut_mux_status in duts_mux_status.items(): + logger.info('Verify that "show mux status" has output ON {}'.format(dut_hostname)) + if len(dut_mux_status) == 0: + err_msg_from_mux_status.append('No mux status in output of "show mux status"') + return False + + logger.info('Verify that mux ports match vlan interfaces of DUT.') + vlan_intf_names = set() + for vlan in duts_minigraph_facts[dut_hostname]['minigraph_vlans'].values(): + vlan_intf_names = vlan_intf_names.union(set(vlan['members'])) + dut_mux_intfs = [] + for row in dut_mux_status: + dut_mux_intfs.append(row['port']) + if vlan_intf_names != set(dut_mux_intfs): + err_msg_from_mux_status.append('Mux ports mismatch vlan interfaces, please check output of "show mux status"') + return False + + logger.info('Verify mux status and parse active/standby side') + dut_parsed_mux_status = {} + for row in dut_mux_status: + # Verify that mux status is either active or standby + if row['status'] not in ['active', 'standby']: + err_msg_from_mux_status.append('Unexpected mux status "{}", please check output of "show mux status"'.format(row['status'])) + return False + + # Parse mux status, transform port name to port index, which is also mux index + port_name = row['port'] + port_idx = duts_minigraph_facts[dut_hostname]['minigraph_port_indices'][port_name] + + # Transform "active" and "standby" to active side which is "upper_tor" or "lower_tor" + status = row['status'] + if dut_hostname == dut_upper_tor.hostname: + # On upper tor, mux status "active" means that active side of mux is upper_tor + # mux status "standby" means that active side of mux is lower_tor + active_side = UPPER_TOR if status == 'active' else LOWER_TOR + else: + # On lower tor, mux status "active" means that active side of mux is lower_tor + # mux status "standby" means that active side of mux is upper_tor + active_side = UPPER_TOR if status == 'standby' else LOWER_TOR + dut_parsed_mux_status[str(port_idx)] = active_side + duts_parsed_mux_status[dut_hostname] = dut_parsed_mux_status + + logger.info('Verify that the mux status on both ToRs are consistent') + upper_tor_mux_status = duts_parsed_mux_status[dut_upper_tor.hostname] + lower_tor_mux_status = duts_parsed_mux_status[dut_lower_tor.hostname] + + logger.info('Verify that mux status is consistent on both ToRs.') + for port_idx in upper_tor_mux_status: + if upper_tor_mux_status[port_idx] != lower_tor_mux_status[port_idx]: + err_msg_from_mux_status.append('Inconsistent mux status on dualtors, please check output of "show mux status"') + return False + + logger.info('Check passed, return parsed mux status') + err_msg_from_mux_status.append("") + return True + dut_upper_tor = duthosts[0] dut_lower_tor = duthosts[1] - # Run "show mux status" on dualtor DUTs to collect mux status - duts_mux_status = duthosts.show_and_parse('show mux status') - - # Parse and basic check duts_parsed_mux_status = {} - for dut_hostname, dut_mux_status in duts_mux_status.items(): - - logger.info('Verify that "show mux status" has output ON {}'.format(dut_hostname)) - if len(dut_mux_status) == 0: - err_msg = 'No mux status in output of "show mux status"' - return False, err_msg, {} - - logger.info('Verify that mux ports match vlan interfaces of DUT.') - vlan_intf_names = set() - for vlan in duts_minigraph_facts[dut_hostname]['minigraph_vlans'].values(): - vlan_intf_names = vlan_intf_names.union(set(vlan['members'])) - dut_mux_intfs = [] - for row in dut_mux_status: - dut_mux_intfs.append(row['port']) - if vlan_intf_names != set(dut_mux_intfs): - err_msg = 'Mux ports mismatch vlan interfaces, please check output of "show mux status"' - return False, err_msg, {} - - logger.info('Verify mux status and parse active/standby side') - dut_parsed_mux_status = {} - for row in dut_mux_status: - # Verify that mux status is either active or standby - if row['status'] not in ['active', 'standby']: - err_msg = 'Unexpected mux status "{}", please check output of "show mux status"'.format(row['status']) - return False, err_msg, {} - - # Parse mux status, transform port name to port index, which is also mux index - port_name = row['port'] - port_idx = duts_minigraph_facts[dut_hostname]['minigraph_port_indices'][port_name] - - # Transform "active" and "standby" to active side which is "upper_tor" or "lower_tor" - status = row['status'] - if dut_hostname == dut_upper_tor.hostname: - # On upper tor, mux status "active" means that active side of mux is upper_tor - # mux status "standby" means that active side of mux is lower_tor - active_side = UPPER_TOR if status == 'active' else LOWER_TOR - else: - # On lower tor, mux status "active" means that active side of mux is lower_tor - # mux status "standby" means that active side of mux is upper_tor - active_side = UPPER_TOR if status == 'standby' else LOWER_TOR - dut_parsed_mux_status[str(port_idx)] = active_side - duts_parsed_mux_status[dut_hostname] = dut_parsed_mux_status - - logger.info('Verify that the mux status on both ToRs are consistent') - upper_tor_mux_status = duts_parsed_mux_status[dut_upper_tor.hostname] - lower_tor_mux_status = duts_parsed_mux_status[dut_lower_tor.hostname] - - logger.info('Verify that mux status is consistent on both ToRs.') - for port_idx in upper_tor_mux_status: - if upper_tor_mux_status[port_idx] != lower_tor_mux_status[port_idx]: - err_msg = 'Inconsistent mux status on dualtors, please check output of "show mux status"' - return False, err_msg, {} - - logger.info('Check passed, return parsed mux status') - return True, "", upper_tor_mux_status + err_msg_from_mux_status = [] + + if not wait_until(30, 5, 0, _verify_show_mux_status): + if err_msg_from_mux_status: + err_msg = err_msg_from_mux_status[-1] + else: + err_msg = "Unknown error occured inside the check" + return False, err_msg, {} + + return True, "", duts_parsed_mux_status @pytest.fixture(scope='module')