diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index 62a142234a..00cc355d83 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -1589,6 +1589,12 @@ qos/test_tunnel_qos_remap.py::test_pfc_watermark_extra_lossless_standby: - "asic_type in ['broadcom']" - https://github.com/sonic-net/sonic-mgmt/issues/11271 +qos/test_voq_watchdog.py: + skip: + reason: "These tests only apply to cisco 8000 platforms." + conditions: + - "asic_type not in ['cisco-8000']" + ####################################### ##### radv ##### ####################################### diff --git a/tests/qos/qos_sai_base.py b/tests/qos/qos_sai_base.py index 4fad7f082a..ec7ee511c0 100644 --- a/tests/qos/qos_sai_base.py +++ b/tests/qos/qos_sai_base.py @@ -11,6 +11,7 @@ import copy import time import collections +from contextlib import contextmanager from tests.common.fixtures.ptfhost_utils import ptf_portmap_file # noqa F401 from tests.common.helpers.assertions import pytest_assert, pytest_require @@ -2736,8 +2737,43 @@ def change_lag_lacp_timer(self, duthosts, get_src_dst_asic_and_duts, tbinfo, nbr "Changing lacp timer multiplier to default for %s in %s" % (neighbor_lag_member, peer_device)) vm_host.no_lacp_time_multiplier(neighbor_lag_member) - @pytest.fixture(scope='class', autouse=True) - def disable_voq_watchdog(self, duthosts, get_src_dst_asic_and_duts, dutConfig): + def get_port_channel_members(self, dut, port_name): + """ + Get the members of portchannel on the given port. + Args: + dut (AnsibleHost): Device Under Test (DUT) + port_name (str): Name of the port to check + Returns: + list: List of port names that are members of the same portchannel. + """ + interfaces = [port_name] + mgfacts = dut.get_extended_minigraph_facts() + for portchan in mgfacts['minigraph_portchannels']: + members = mgfacts['minigraph_portchannels'][portchan]['members'] + if port_name in members: + logger.info("Interface {} is a member of portchannel {}, setting interface list to members {}".format( + port_name, portchan, members)) + interfaces = members + break + return interfaces + + def voq_watchdog_enabled(self, get_src_dst_asic_and_duts): + dst_dut = get_src_dst_asic_and_duts['dst_dut'] + if not is_cisco_device(dst_dut): + return False + namespace_option = "-n asic0" if dst_dut.facts.get("modular_chassis") else "" + show_command = "show platform npu global {}".format(namespace_option) + result = run_dshell_command(dst_dut, show_command) + pattern = r"voq_watchdog_enabled +: +True" + match = re.search(pattern, result["stdout"]) + return match + + def modify_voq_watchdog(self, duthosts, get_src_dst_asic_and_duts, dutConfig, enable): + # Skip if voq watchdog is not enabled. + if not self.voq_watchdog_enabled(get_src_dst_asic_and_duts): + logger.info("voq_watchdog is not enabled, skipping modify voq watchdog") + return + dst_dut = get_src_dst_asic_and_duts['dst_dut'] dst_asic = get_src_dst_asic_and_duts['dst_asic'] dut_list = [dst_dut] @@ -2754,55 +2790,34 @@ def disable_voq_watchdog(self, duthosts, get_src_dst_asic_and_duts, dutConfig): dut_list.append(rp_dut) asic_index_list.append(asic.asic_index) - if dst_dut.facts['asic_type'] != "cisco-8000" or not dst_dut.sonichost.is_multi_asic: - yield - return - - # Disable voq watchdog. + # Modify voq watchdog. for (dut, asic_index) in zip(dut_list, asic_index_list): copy_set_voq_watchdog_script_cisco_8000( dut=dut, asic=asic_index, - enable=False) + enable=enable) cmd_opt = "-n asic{}".format(asic_index) if not dst_dut.sonichost.is_multi_asic: cmd_opt = "" dut.shell("sudo show platform npu script {} -s set_voq_watchdog.py".format(cmd_opt)) + @contextmanager + def disable_voq_watchdog(self, duthosts, get_src_dst_asic_and_duts, dutConfig): + # Disable voq watchdog. + self.modify_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig, enable=False) yield - # Enable voq watchdog. - for (dut, asic_index) in zip(dut_list, asic_index_list): - copy_set_voq_watchdog_script_cisco_8000( - dut=dut, - asic=asic_index, - enable=True) - cmd_opt = "-n asic{}".format(asic_index) - if not dst_dut.sonichost.is_multi_asic: - cmd_opt = "" - dut.shell("sudo show platform npu script {} -s set_voq_watchdog.py".format(cmd_opt)) + self.modify_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig, enable=True) - return + @pytest.fixture(scope='function') + def disable_voq_watchdog_function_scope(self, duthosts, get_src_dst_asic_and_duts, dutConfig): + with self.disable_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig) as result: + yield result - def get_port_channel_members(self, dut, port_name): - """ - Get the members of portchannel on the given port. - Args: - dut (AnsibleHost): Device Under Test (DUT) - port_name (str): Name of the port to check - Returns: - list: List of port names that are members of the same portchannel. - """ - interfaces = [port_name] - mgfacts = dut.get_extended_minigraph_facts() - for portchan in mgfacts['minigraph_portchannels']: - members = mgfacts['minigraph_portchannels'][portchan]['members'] - if port_name in members: - logger.info("Interface {} is a member of portchannel {}, setting interface list to members {}".format( - port_name, portchan, members)) - interfaces = members - break - return interfaces + @pytest.fixture(scope='class') + def disable_voq_watchdog_class_scope(self, duthosts, get_src_dst_asic_and_duts, dutConfig): + with self.disable_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig) as result: + yield result def oq_watchdog_enabled(self, get_src_dst_asic_and_duts): dst_dut = get_src_dst_asic_and_duts['dst_dut'] diff --git a/tests/qos/test_qos_sai.py b/tests/qos/test_qos_sai.py index 7c35802ce7..b711dd8a1a 100644 --- a/tests/qos/test_qos_sai.py +++ b/tests/qos/test_qos_sai.py @@ -162,6 +162,10 @@ class TestQosSai(QosSaiBase): 'Arista-7050CX3-32S-D48C8' ] + @pytest.fixture(scope="class", autouse=True) + def setup(self, disable_voq_watchdog_class_scope): + return + @pytest.fixture(scope='function') def change_port_speed( self, request, ptfhost, duthosts, dutTestParams, fanouthosts, dutConfig, tbinfo, diff --git a/tests/qos/test_voq_watchdog.py b/tests/qos/test_voq_watchdog.py new file mode 100644 index 0000000000..a570765e01 --- /dev/null +++ b/tests/qos/test_voq_watchdog.py @@ -0,0 +1,109 @@ +"""SAI thrift-based tests for the VOQ watchdog feature in SONiC. + +This set of test cases verifies VOQ watchdog behavior. These are dataplane +tests that depend on the SAI thrift library in order to pause ports and read +drop counters. + +Parameters: + --ptf_portmap (str): file name of port index to DUT interface alias map. Default is None. + In case a filename is not provided, a file containing a port indices to aliases map will be generated. + + --qos_swap_syncd (bool): Used to install the RPC syncd image before running the tests. Default is True. + + --qos_dst_ports (list) Indices of available DUT test ports to serve as destination ports. Note: This is not port + index on DUT, rather an index into filtered (excludes lag member ports) DUT ports. Plan is to randomize port + selection. Default is [0, 1, 3]. + + --qos_src_ports (list) Indices of available DUT test ports to serve as source port. Similar note as in + qos_dst_ports applies. Default is [2]. +""" + +import logging +import pytest + +from tests.common.fixtures.duthost_utils import dut_qos_maps, \ + separated_dscp_to_tc_map_on_uplink # noqa: F401 +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa: F401 +from tests.common.fixtures.ptfhost_utils import copy_saitests_directory # noqa: F401 +from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa: F401 +from .qos_sai_base import QosSaiBase + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('any') +] + +PKTS_NUM = 100 + + +@pytest.fixture(scope="function", autouse=True) +def ignore_log_voq_watchdog(duthosts, loganalyzer): + if not loganalyzer: + yield + return + ignore_list = [r".*HARDWARE_WATCHDOG.*", r".*soft_reset*", r".*VOQ Appears to be stuck*"] + for dut in duthosts: + for line in ignore_list: + loganalyzer[dut.hostname].ignore_regex.append(line) + yield + return + + +class TestVoqWatchdog(QosSaiBase): + """TestVoqWatchdog derives from QosSaiBase and contains collection of VOQ watchdog test cases. + """ + @pytest.fixture(scope="class", autouse=True) + def check_skip_voq_watchdog_test(self, get_src_dst_asic_and_duts): + if not self.voq_watchdog_enabled(get_src_dst_asic_and_duts): + pytest.skip("Voq watchdog test is skipped since voq watchdog is not enabled.") + + @pytest.mark.parametrize("voq_watchdog_enabled", [True, False]) + def testVoqWatchdog( + self, ptfhost, dutTestParams, dutConfig, dutQosConfig, + duthosts, get_src_dst_asic_and_duts, voq_watchdog_enabled + ): + """ + Test VOQ watchdog + Args: + ptfhost (AnsibleHost): Packet Test Framework (PTF) + dutTestParams (Fixture, dict): DUT host test params + dutConfig (Fixture, dict): Map of DUT config containing dut interfaces, test port IDs, test port IPs, + and test ports + dutQosConfig (Fixture, dict): Map containing DUT host QoS configuration + voq_watchdog_enabled (bool): if VOQ watchdog is enabled or not + Returns: + None + Raises: + RunAnsibleModuleFail if ptf test fails + """ + + try: + if not voq_watchdog_enabled: + self.modify_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig, enable=False) + + testParams = dict() + testParams.update(dutTestParams["basicParams"]) + testParams.update({ + "dscp": 8, + "dst_port_id": dutConfig["testPorts"]["dst_port_id"], + "dst_port_ip": dutConfig["testPorts"]["dst_port_ip"], + "src_port_id": dutConfig["testPorts"]["src_port_id"], + "src_port_ip": dutConfig["testPorts"]["src_port_ip"], + "src_port_vlan": dutConfig["testPorts"]["src_port_vlan"], + "packet_size": 1350, + "pkts_num": PKTS_NUM, + "voq_watchdog_enabled": voq_watchdog_enabled, + }) + + self.runPtfTest( + ptfhost, testCase="sai_qos_tests.VoqWatchdogTest", + testParams=testParams) + + self.runPtfTest( + ptfhost, testCase="sai_qos_tests.TrafficSanityTest", + testParams=testParams) + + finally: + if not voq_watchdog_enabled: + self.modify_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig, enable=True) diff --git a/tests/saitests/py3/sai_qos_tests.py b/tests/saitests/py3/sai_qos_tests.py index a473921a59..eb0808108f 100755 --- a/tests/saitests/py3/sai_qos_tests.py +++ b/tests/saitests/py3/sai_qos_tests.py @@ -6250,6 +6250,76 @@ def findFaultySrcDstPair(dscp, queue): assert len(failed_pairs) == 0, "Traffic failed between {}".format(failed_pairs) +class VoqWatchdogTest(sai_base_test.ThriftInterfaceDataPlane): + def runTest(self): + switch_init(self.clients) + + # Parse input parameters + dscp = int(self.test_params['dscp']) + router_mac = self.test_params['router_mac'] + sonic_version = self.test_params['sonic_version'] + dst_port_id = int(self.test_params['dst_port_id']) + dst_port_ip = self.test_params['dst_port_ip'] + dst_port_mac = self.dataplane.get_mac(0, dst_port_id) + src_port_id = int(self.test_params['src_port_id']) + src_port_ip = self.test_params['src_port_ip'] + src_port_vlan = self.test_params['src_port_vlan'] + src_port_mac = self.dataplane.get_mac(0, src_port_id) + voq_watchdog_enabled = self.test_params['voq_watchdog_enabled'] + asic_type = self.test_params['sonic_asic_type'] + pkts_num = int(self.test_params['pkts_num']) + + pkt_dst_mac = router_mac if router_mac != '' else dst_port_mac + # get counter names to query + ingress_counters, egress_counters = get_counter_names(sonic_version) + + # Prepare IP packet data + ttl = 64 + if 'packet_size' in list(self.test_params.keys()): + packet_length = int(self.test_params['packet_size']) + else: + packet_length = 64 + + is_dualtor = self.test_params.get('is_dualtor', False) + def_vlan_mac = self.test_params.get('def_vlan_mac', None) + if is_dualtor and def_vlan_mac is not None: + pkt_dst_mac = def_vlan_mac + + pkt = construct_ip_pkt(packet_length, + pkt_dst_mac, + src_port_mac, + src_port_ip, + dst_port_ip, + dscp, + src_port_vlan, + ttl=ttl) + + log_message("test dst_port_id: {}, src_port_id: {}, src_vlan: {}".format( + dst_port_id, src_port_id, src_port_vlan), to_stderr=True) + # in case dst_port_id is part of LAG, find out the actual dst port + # for given IP parameters + dst_port_id = get_rx_port( + self, 0, src_port_id, pkt_dst_mac, dst_port_ip, src_port_ip, src_port_vlan + ) + log_message("actual dst_port_id: {}".format(dst_port_id), to_stderr=True) + + self.sai_thrift_port_tx_disable(self.dst_client, asic_type, [dst_port_id]) + pre_offsets = init_log_check(self) + + try: + # send packets + send_packet(self, src_port_id, pkt, pkts_num) + + # allow enough time to trigger voq watchdog + time.sleep(WATCHDOG_TIMEOUT_SECONDS["voq"] * 1.3) + + # verify voq watchdog is triggered + verify_log(self, pre_offsets, voq_watchdog_enabled, "voq") + + finally: + self.sai_thrift_port_tx_enable(self.dst_client, asic_type, [dst_port_id]) + + class OqWatchdogTest(sai_base_test.ThriftInterfaceDataPlane): def runTest(self): switch_init(self.clients)