[202405] New test cases to verify voq watchdog

tests/common/plugins/conditional_mark/tests_mark_conditions.yaml

@@ -1589,6 +1589,12 @@ qos/test_tunnel_qos_remap.py::test_pfc_watermark_extra_lossless_standby:
       - "asic_type in ['broadcom']"
       - https://github.com/sonic-net/sonic-mgmt/issues/11271
 
+qos/test_voq_watchdog.py:
+  skip:
+    reason: "These tests only apply to cisco 8000 platforms."
+    conditions:
+      - "asic_type not in ['cisco-8000']"
+
 #######################################
 #####           radv             #####
 #######################################

tests/qos/qos_sai_base.py

@@ -11,6 +11,7 @@
 import copy
 import time
 import collections
+from contextlib import contextmanager
 
 from tests.common.fixtures.ptfhost_utils import ptf_portmap_file  # noqa F401
 from tests.common.helpers.assertions import pytest_assert, pytest_require
@@ -2736,8 +2737,43 @@ def change_lag_lacp_timer(self, duthosts, get_src_dst_asic_and_duts, tbinfo, nbr
                         "Changing lacp timer multiplier to default for %s in %s" % (neighbor_lag_member, peer_device))
                     vm_host.no_lacp_time_multiplier(neighbor_lag_member)
 
-    @pytest.fixture(scope='class', autouse=True)
-    def disable_voq_watchdog(self, duthosts, get_src_dst_asic_and_duts, dutConfig):
+    def get_port_channel_members(self, dut, port_name):
+        """
+        Get the members of portchannel on the given port.
+        Args:
+            dut (AnsibleHost): Device Under Test (DUT)
+            port_name (str): Name of the port to check
+        Returns:
+            list: List of port names that are members of the same portchannel.
+        """
+        interfaces = [port_name]
+        mgfacts = dut.get_extended_minigraph_facts()
+        for portchan in mgfacts['minigraph_portchannels']:
+            members = mgfacts['minigraph_portchannels'][portchan]['members']
+            if port_name in members:
+                logger.info("Interface {} is a member of portchannel {}, setting interface list to members {}".format(
+                    port_name, portchan, members))
+                interfaces = members
+                break
+        return interfaces
+
+    def voq_watchdog_enabled(self, get_src_dst_asic_and_duts):
+        dst_dut = get_src_dst_asic_and_duts['dst_dut']
+        if not is_cisco_device(dst_dut):
+            return False
+        namespace_option = "-n asic0" if dst_dut.facts.get("modular_chassis") else ""
+        show_command = "show platform npu global {}".format(namespace_option)
+        result = run_dshell_command(dst_dut, show_command)
+        pattern = r"voq_watchdog_enabled +: +True"
+        match = re.search(pattern, result["stdout"])
+        return match
+
+    def modify_voq_watchdog(self, duthosts, get_src_dst_asic_and_duts, dutConfig, enable):
+        # Skip if voq watchdog is not enabled.
+        if not self.voq_watchdog_enabled(get_src_dst_asic_and_duts):
+            logger.info("voq_watchdog is not enabled, skipping modify voq watchdog")
+            return
+
         dst_dut = get_src_dst_asic_and_duts['dst_dut']
         dst_asic = get_src_dst_asic_and_duts['dst_asic']
         dut_list = [dst_dut]
@@ -2754,55 +2790,34 @@ def disable_voq_watchdog(self, duthosts, get_src_dst_asic_and_duts, dutConfig):
                     dut_list.append(rp_dut)
                     asic_index_list.append(asic.asic_index)
 
-        if dst_dut.facts['asic_type'] != "cisco-8000" or not dst_dut.sonichost.is_multi_asic:
-            yield
-            return
-
-        # Disable voq watchdog.
+        # Modify voq watchdog.
         for (dut, asic_index) in zip(dut_list, asic_index_list):
             copy_set_voq_watchdog_script_cisco_8000(
                 dut=dut,
                 asic=asic_index,
-                enable=False)
+                enable=enable)
             cmd_opt = "-n asic{}".format(asic_index)
             if not dst_dut.sonichost.is_multi_asic:
                 cmd_opt = ""
             dut.shell("sudo show platform npu script {} -s set_voq_watchdog.py".format(cmd_opt))
 
+    @contextmanager
+    def disable_voq_watchdog(self, duthosts, get_src_dst_asic_and_duts, dutConfig):
+        # Disable voq watchdog.
+        self.modify_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig, enable=False)
         yield
-
         # Enable voq watchdog.
-        for (dut, asic_index) in zip(dut_list, asic_index_list):
-            copy_set_voq_watchdog_script_cisco_8000(
-                dut=dut,
-                asic=asic_index,
-                enable=True)
-            cmd_opt = "-n asic{}".format(asic_index)
-            if not dst_dut.sonichost.is_multi_asic:
-                cmd_opt = ""
-            dut.shell("sudo show platform npu script {} -s set_voq_watchdog.py".format(cmd_opt))
+        self.modify_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig, enable=True)
 
-        return
+    @pytest.fixture(scope='function')
+    def disable_voq_watchdog_function_scope(self, duthosts, get_src_dst_asic_and_duts, dutConfig):
+        with self.disable_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig) as result:
+            yield result
 
-    def get_port_channel_members(self, dut, port_name):
-        """
-        Get the members of portchannel on the given port.
-        Args:
-            dut (AnsibleHost): Device Under Test (DUT)
-            port_name (str): Name of the port to check
-        Returns:
-            list: List of port names that are members of the same portchannel.
-        """
-        interfaces = [port_name]
-        mgfacts = dut.get_extended_minigraph_facts()
-        for portchan in mgfacts['minigraph_portchannels']:
-            members = mgfacts['minigraph_portchannels'][portchan]['members']
-            if port_name in members:
-                logger.info("Interface {} is a member of portchannel {}, setting interface list to members {}".format(
-                    port_name, portchan, members))
-                interfaces = members
-                break
-        return interfaces
+    @pytest.fixture(scope='class')
+    def disable_voq_watchdog_class_scope(self, duthosts, get_src_dst_asic_and_duts, dutConfig):
+        with self.disable_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig) as result:
+            yield result
 
     def oq_watchdog_enabled(self, get_src_dst_asic_and_duts):
         dst_dut = get_src_dst_asic_and_duts['dst_dut']

tests/qos/test_qos_sai.py

@@ -162,6 +162,10 @@ class TestQosSai(QosSaiBase):
         'Arista-7050CX3-32S-D48C8'
     ]
 
+    @pytest.fixture(scope="class", autouse=True)
+    def setup(self, disable_voq_watchdog_class_scope):
+        return
+
     @pytest.fixture(scope='function')
     def change_port_speed(
             self, request, ptfhost, duthosts, dutTestParams, fanouthosts, dutConfig, tbinfo,

tests/qos/test_voq_watchdog.py

@@ -0,0 +1,109 @@
+"""SAI thrift-based tests for the VOQ watchdog feature in SONiC.
+
+This set of test cases verifies VOQ watchdog behavior. These are dataplane
+tests that depend on the SAI thrift library in order to pause ports and read
+drop counters.
+
+Parameters:
+    --ptf_portmap <filename> (str): file name of port index to DUT interface alias map. Default is None.
+        In case a filename is not provided, a file containing a port indices to aliases map will be generated.
+
+    --qos_swap_syncd (bool): Used to install the RPC syncd image before running the tests. Default is True.
+
+    --qos_dst_ports (list) Indices of available DUT test ports to serve as destination ports. Note: This is not port
+        index on DUT, rather an index into filtered (excludes lag member ports) DUT ports. Plan is to randomize port
+        selection. Default is [0, 1, 3].
+
+    --qos_src_ports (list) Indices of available DUT test ports to serve as source port. Similar note as in
+        qos_dst_ports applies. Default is [2].
+"""
+
+import logging
+import pytest
+
+from tests.common.fixtures.duthost_utils import dut_qos_maps, \
+    separated_dscp_to_tc_map_on_uplink                                                      # noqa: F401
+from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory                     # noqa: F401
+from tests.common.fixtures.ptfhost_utils import copy_saitests_directory                     # noqa: F401
+from tests.common.fixtures.ptfhost_utils import change_mac_addresses                        # noqa: F401
+from .qos_sai_base import QosSaiBase
+
+logger = logging.getLogger(__name__)
+
+pytestmark = [
+    pytest.mark.topology('any')
+]
+
+PKTS_NUM = 100
+
+
+@pytest.fixture(scope="function", autouse=True)
+def ignore_log_voq_watchdog(duthosts, loganalyzer):
+    if not loganalyzer:
+        yield
+        return
+    ignore_list = [r".*HARDWARE_WATCHDOG.*", r".*soft_reset*", r".*VOQ Appears to be stuck*"]
+    for dut in duthosts:
+        for line in ignore_list:
+            loganalyzer[dut.hostname].ignore_regex.append(line)
+    yield
+    return
+
+
+class TestVoqWatchdog(QosSaiBase):
+    """TestVoqWatchdog derives from QosSaiBase and contains collection of VOQ watchdog test cases.
+    """
+    @pytest.fixture(scope="class", autouse=True)
+    def check_skip_voq_watchdog_test(self, get_src_dst_asic_and_duts):
+        if not self.voq_watchdog_enabled(get_src_dst_asic_and_duts):
+            pytest.skip("Voq watchdog test is skipped since voq watchdog is not enabled.")
+
+    @pytest.mark.parametrize("voq_watchdog_enabled", [True, False])
+    def testVoqWatchdog(
+            self, ptfhost, dutTestParams, dutConfig, dutQosConfig,
+            duthosts, get_src_dst_asic_and_duts, voq_watchdog_enabled
+    ):
+        """
+            Test VOQ watchdog
+            Args:
+                ptfhost (AnsibleHost): Packet Test Framework (PTF)
+                dutTestParams (Fixture, dict): DUT host test params
+                dutConfig (Fixture, dict): Map of DUT config containing dut interfaces, test port IDs, test port IPs,
+                    and test ports
+                dutQosConfig (Fixture, dict): Map containing DUT host QoS configuration
+                voq_watchdog_enabled (bool): if VOQ watchdog is enabled or not
+            Returns:
+                None
+            Raises:
+                RunAnsibleModuleFail if ptf test fails
+        """
+
+        try:
+            if not voq_watchdog_enabled:
+                self.modify_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig, enable=False)
+
+            testParams = dict()
+            testParams.update(dutTestParams["basicParams"])
+            testParams.update({
+                "dscp": 8,
+                "dst_port_id": dutConfig["testPorts"]["dst_port_id"],
+                "dst_port_ip": dutConfig["testPorts"]["dst_port_ip"],
+                "src_port_id": dutConfig["testPorts"]["src_port_id"],
+                "src_port_ip": dutConfig["testPorts"]["src_port_ip"],
+                "src_port_vlan": dutConfig["testPorts"]["src_port_vlan"],
+                "packet_size": 1350,
+                "pkts_num": PKTS_NUM,
+                "voq_watchdog_enabled": voq_watchdog_enabled,
+            })
+
+            self.runPtfTest(
+                ptfhost, testCase="sai_qos_tests.VoqWatchdogTest",
+                testParams=testParams)
+
+            self.runPtfTest(
+                ptfhost, testCase="sai_qos_tests.TrafficSanityTest",
+                testParams=testParams)
+
+        finally:
+            if not voq_watchdog_enabled:
+                self.modify_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig, enable=True)

tests/saitests/py3/sai_qos_tests.py

@@ -6250,6 +6250,76 @@ def findFaultySrcDstPair(dscp, queue):
         assert len(failed_pairs) == 0, "Traffic failed between {}".format(failed_pairs)
 
 
+class VoqWatchdogTest(sai_base_test.ThriftInterfaceDataPlane):
+    def runTest(self):
+        switch_init(self.clients)
+
+        # Parse input parameters
+        dscp = int(self.test_params['dscp'])
+        router_mac = self.test_params['router_mac']
+        sonic_version = self.test_params['sonic_version']
+        dst_port_id = int(self.test_params['dst_port_id'])
+        dst_port_ip = self.test_params['dst_port_ip']
+        dst_port_mac = self.dataplane.get_mac(0, dst_port_id)
+        src_port_id = int(self.test_params['src_port_id'])
+        src_port_ip = self.test_params['src_port_ip']
+        src_port_vlan = self.test_params['src_port_vlan']
+        src_port_mac = self.dataplane.get_mac(0, src_port_id)
+        voq_watchdog_enabled = self.test_params['voq_watchdog_enabled']
+        asic_type = self.test_params['sonic_asic_type']
+        pkts_num = int(self.test_params['pkts_num'])
+
+        pkt_dst_mac = router_mac if router_mac != '' else dst_port_mac
+        # get counter names to query
+        ingress_counters, egress_counters = get_counter_names(sonic_version)
+
+        # Prepare IP packet data
+        ttl = 64
+        if 'packet_size' in list(self.test_params.keys()):
+            packet_length = int(self.test_params['packet_size'])
+        else:
+            packet_length = 64
+
+        is_dualtor = self.test_params.get('is_dualtor', False)
+        def_vlan_mac = self.test_params.get('def_vlan_mac', None)
+        if is_dualtor and def_vlan_mac is not None:
+            pkt_dst_mac = def_vlan_mac
+
+        pkt = construct_ip_pkt(packet_length,
+                               pkt_dst_mac,
+                               src_port_mac,
+                               src_port_ip,
+                               dst_port_ip,
+                               dscp,
+                               src_port_vlan,
+                               ttl=ttl)
+
+        log_message("test dst_port_id: {}, src_port_id: {}, src_vlan: {}".format(
+            dst_port_id, src_port_id, src_port_vlan), to_stderr=True)
+        # in case dst_port_id is part of LAG, find out the actual dst port
+        # for given IP parameters
+        dst_port_id = get_rx_port(
+            self, 0, src_port_id, pkt_dst_mac, dst_port_ip, src_port_ip, src_port_vlan
+        )
+        log_message("actual dst_port_id: {}".format(dst_port_id), to_stderr=True)
+
+        self.sai_thrift_port_tx_disable(self.dst_client, asic_type, [dst_port_id])
+        pre_offsets = init_log_check(self)
+
+        try:
+            # send packets
+            send_packet(self, src_port_id, pkt, pkts_num)
+
+            # allow enough time to trigger voq watchdog
+            time.sleep(WATCHDOG_TIMEOUT_SECONDS["voq"] * 1.3)
+
+            # verify voq watchdog is triggered
+            verify_log(self, pre_offsets, voq_watchdog_enabled, "voq")
+
+        finally:
+            self.sai_thrift_port_tx_enable(self.dst_client, asic_type, [dst_port_id])
+
+
 class OqWatchdogTest(sai_base_test.ThriftInterfaceDataPlane):
     def runTest(self):
         switch_init(self.clients)