New test case for OQ watchdog

tests/common/cisco_data.py

@@ -1,6 +1,7 @@
 import json
 import re
 from tests.common.reboot import reboot
+from tests.common.utilities import wait_until
 
 
 def is_cisco_device(dut):
@@ -97,3 +98,18 @@ def set_voq_watchdog(enable):
 '''.format(enable)
 
     copy_dshell_script_cisco_8000(dut, asic, dshell_script, script_name="set_voq_watchdog.py")
+
+
+def check_dshell_ready(duthost):
+    show_command = "sudo show platform npu rx cgm_global"
+    err_msg = "debug shell server for asic 0 is not running"
+    output = duthost.command(show_command)['stdout']
+    if err_msg in output:
+        return False
+    return True
+
+
+def run_dshell_command(duthost, command):
+    if not wait_until(300, 20, 0, check_dshell_ready, duthost):
+        raise RuntimeError("Debug shell is not ready on {}".format(duthost.hostname))
+    return duthost.shell(command)

tests/common/plugins/conditional_mark/tests_mark_conditions.yaml

@@ -1921,6 +1921,12 @@ qos/test_ecn_config.py:
     conditions:
       - "asic_type not in ['cisco-8000']"
 
+qos/test_oq_watchdog.py:
+  skip:
+    reason: "OQ watchdog tests only apply to cisco 8000 platforms."
+    conditions:
+      - "asic_type not in ['cisco-8000']"
+
 qos/test_pfc_pause.py::test_pfc_pause_lossless:
   # For this test, we use the fanout connected to the DUT to send PFC pause frames.
   # The fanout needs to send PFC frames fast enough so that the queue remains completely paused for the entire duration

tests/qos/qos_sai_base.py

@@ -18,7 +18,7 @@
 from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor
 from tests.common.mellanox_data import is_mellanox_device as isMellanoxDevice
 from tests.common.cisco_data import is_cisco_device, copy_set_voq_watchdog_script_cisco_8000, \
-        copy_dshell_script_cisco_8000
+        copy_dshell_script_cisco_8000, run_dshell_command
 from tests.common.dualtor.dual_tor_common import active_standby_ports  # noqa F401
 from tests.common.dualtor.dual_tor_utils import upper_tor_host, lower_tor_host, dualtor_ports, is_tunnel_qos_remap_enabled  # noqa F401
 from tests.common.dualtor.mux_simulator_control \
@@ -2782,6 +2782,26 @@ def set_port_cir(interface, rate):
 
         copy_dshell_script_cisco_8000(dut, asic, dshell_script, script_name="set_scheduler.py")
 
+    def get_port_channel_members(self, dut, port_name):
+        """
+        Get the members of portchannel on the given port.
+        Args:
+            dut (AnsibleHost): Device Under Test (DUT)
+            port_name (str): Name of the port to check
+        Returns:
+            list: List of port names that are members of the same portchannel.
+        """
+        interfaces = [port_name]
+        mgfacts = dut.get_extended_minigraph_facts()
+        for portchan in mgfacts['minigraph_portchannels']:
+            members = mgfacts['minigraph_portchannels'][portchan]['members']
+            if port_name in members:
+                logger.info("Interface {} is a member of portchannel {}, setting interface list to members {}".format(
+                    port_name, portchan, members))
+                interfaces = members
+                break
+        return interfaces
+
     @pytest.fixture(scope="function", autouse=False)
     def set_cir_change(self, get_src_dst_asic_and_duts, dutConfig):
         dst_port = dutConfig['dutInterfaces'][dutConfig["testPorts"]["dst_port_id"]]
@@ -2793,16 +2813,7 @@ def set_cir_change(self, get_src_dst_asic_and_duts, dutConfig):
             yield
             return
 
-        interfaces = [dst_port]
-        # If interface is a PortChannel member, expand the list of interfaces that need scheduler setting
-        mgfacts = dst_dut.get_extended_minigraph_facts()
-        for portchan in mgfacts['minigraph_portchannels']:
-            members = mgfacts['minigraph_portchannels'][portchan]['members']
-            if dst_port in members:
-                logger.info("Interface {} is a member of portchannel {}, setting interfaces list to members {}".format(
-                    dst_port, portchan, members))
-                interfaces = members
-                break
+        interfaces = self.get_port_channel_members(dst_dut, dst_port)
 
         # Set scheduler to 5 Gbps.
         self.copy_set_cir_script_cisco_8000(
@@ -2816,11 +2827,11 @@ def set_cir_change(self, get_src_dst_asic_and_duts, dutConfig):
 
     def voq_watchdog_enabled(self, get_src_dst_asic_and_duts):
         dst_dut = get_src_dst_asic_and_duts['dst_dut']
-        if dst_dut.facts['asic_type'] != "cisco-8000":
+        if not is_cisco_device(dst_dut):
             return False
         namespace_option = "-n asic0" if dst_dut.facts.get("modular_chassis") else ""
         show_command = "show platform npu global {}".format(namespace_option)
-        result = dst_dut.command(show_command)
+        result = run_dshell_command(dst_dut, show_command)
         pattern = r"voq_watchdog_enabled +: +True"
         match = re.search(pattern, result["stdout"])
         return match
@@ -2882,3 +2893,64 @@ def disable_voq_watchdog_function_scope(self, duthosts, get_src_dst_asic_and_dut
     def disable_voq_watchdog_class_scope(self, duthosts, get_src_dst_asic_and_duts, dutConfig):
         with self.disable_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig) as result:
             yield result
+
+    def oq_watchdog_enabled(self, get_src_dst_asic_and_duts):
+        dst_dut = get_src_dst_asic_and_duts['dst_dut']
+        if not is_cisco_device(dst_dut):
+            return False
+        namespace_option = "-n asic0" if dst_dut.facts.get("modular_chassis") else ""
+        show_command = "show platform npu global {}".format(namespace_option)
+        result = run_dshell_command(dst_dut, show_command)
+        pattern = r"gb_watchdog_oq_enabled +: +True"
+        match = re.search(pattern, result["stdout"])
+        return match
+
+    def copy_set_queue_pir_script_cisco_8000(self, dut, ports, queue, speed, schduler_type, asic=""):
+        dshell_script = '''
+from common import *
+def set_queue_pir(interface, queue, rate):
+    sai_lane = port_to_sai_lane_map[interface]
+    slice_id, ifg_id, serdes_id = sai_lane_to_slice_ifg_serdes(sai_lane)
+    sys_port = find_system_port(slice_id, ifg_id, serdes_id)
+    scheduler = sys_port.get_scheduler()
+    original_pir = scheduler.get_{}_pir(queue)
+    print(original_pir)
+    scheduler.set_{}_pir(queue, rate)
+'''.format(schduler_type, schduler_type)
+
+        for intf in ports:
+            dshell_script += 'set_queue_pir("{}", {}, {})\n'.format(intf, queue, speed)
+
+        copy_dshell_script_cisco_8000(dut, asic, dshell_script, script_name="set_queue_pir.py")
+
+    def block_queue(self, dut, port, queue, queue_type, asic_index=""):
+        interfaces = self.get_port_channel_members(dut, port)
+        scheduler_type = "credit" if queue_type == "voq" else "transmit"
+        self.copy_set_queue_pir_script_cisco_8000(
+            dut=dut,
+            ports=interfaces,
+            queue=queue,
+            asic=asic_index,
+            speed=0,
+            schduler_type=scheduler_type)
+        cmd_opt = "-n asic{}".format(asic_index)
+        if not dut.sonichost.is_multi_asic:
+            cmd_opt = ""
+        result = dut.shell("sudo show platform npu script {} -s set_queue_pir.py".format(cmd_opt))
+        original_pir = result["stdout_lines"][0]
+        return int(original_pir)
+
+    def unblock_queue(self, dut, port, queue, queue_type, speed, asic_index=""):
+        interfaces = self.get_port_channel_members(dut, port)
+        scheduler_type = "credit" if queue_type == "voq" else "transmit"
+        self.copy_set_queue_pir_script_cisco_8000(
+            dut=dut,
+            ports=interfaces,
+            queue=queue,
+            asic=asic_index,
+            speed=speed,
+            schduler_type=scheduler_type)
+        cmd_opt = "-n asic{}".format(asic_index)
+        if not dut.sonichost.is_multi_asic:
+            cmd_opt = ""
+        dut.shell("sudo show platform npu script {} -s set_queue_pir.py".format(cmd_opt))

tests/qos/test_oq_watchdog.py

@@ -0,0 +1,125 @@
+"""SAI thrift-based tests for the OQ watchdog feature in SONiC.
+
+This set of test cases verifies OQ watchdog behavior. These are dataplane
+tests that depend on the SAI thrift library in order to pause ports and read
+drop counters.
+
+Parameters:
+    --ptf_portmap <filename> (str): file name of port index to DUT interface alias map. Default is None.
+        In case a filename is not provided, a file containing a port indices to aliases map will be generated.
+
+    --qos_swap_syncd (bool): Used to install the RPC syncd image before running the tests. Default is True.
+
+    --qos_dst_ports (list) Indices of available DUT test ports to serve as destination ports. Note: This is not port
+        index on DUT, rather an index into filtered (excludes lag member ports) DUT ports. Plan is to randomize port
+        selection. Default is [0, 1, 3].
+
+    --qos_src_ports (list) Indices of available DUT test ports to serve as source port. Similar note as in
+        qos_dst_ports applies. Default is [2].
+"""
+
+import logging
+import pytest
+
+from tests.common.fixtures.duthost_utils import dut_qos_maps, \
+    separated_dscp_to_tc_map_on_uplink                                                      # noqa F401
+from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory                     # noqa F401
+from tests.common.fixtures.ptfhost_utils import copy_saitests_directory                     # noqa F401
+from tests.common.fixtures.ptfhost_utils import change_mac_addresses                        # noqa F401
+from .qos_sai_base import QosSaiBase
+
+logger = logging.getLogger(__name__)
+
+pytestmark = [
+    pytest.mark.topology('any')
+]
+
+PKTS_NUM = 100
+
+
+@pytest.fixture(scope="function")
+def ignore_log_oq_watchdog(duthosts, loganalyzer):
+    if not loganalyzer:
+        yield
+        return
+    ignore_list = [r".*HARDWARE_WATCHDOG.*", r".*soft_reset*"]
+    for dut in duthosts:
+        for line in ignore_list:
+            loganalyzer[dut.hostname].ignore_regex.append(line)
+    yield
+    return
+
+
+class TestOqWatchdog(QosSaiBase):
+    """TestVoqWatchdog derives from QosSaiBase and contains collection of OQ watchdog test cases.
+    """
+    @pytest.fixture(scope="class", autouse=True)
+    def check_skip_oq_watchdog_test(self, get_src_dst_asic_and_duts):
+        if not self.oq_watchdog_enabled(get_src_dst_asic_and_duts):
+            pytest.skip("OQ watchdog test is skipped since OQ watchdog is not enabled.")
+
+    def testOqWatchdog(
+            self, ptfhost, dutTestParams, dutConfig, dutQosConfig,
+            get_src_dst_asic_and_duts, ignore_log_oq_watchdog,
+            disable_voq_watchdog_function_scope
+    ):
+        """
+            Test OQ watchdog functionality.
+            Test steps:
+                1. block voq7, sys_port scheduler set Q7 credit_pir to 0
+                2. fill leakout of Q7 by ping, make sure no packet dequeue/enqueue in OQ7 afterwards
+                3. block oq0, sys_port scheduler set Q0 transmit_pir to 0
+                4. send traffic on Q0, oq watchdog should be triggered in about 5 seconds
+                5. Unblock voq7 and oq0 to restore the system state
+                6. Run TrafficSanityTest to verify the system state is restored
+            Args:
+                ptfhost (AnsibleHost): Packet Test Framework (PTF)
+                dutTestParams (Fixture, dict): DUT host test params
+                dutConfig (Fixture, dict): Map of DUT config containing dut interfaces, test port IDs, test port IPs,
+                    and test ports
+                dutQosConfig (Fixture, dict): Map containing DUT host QoS configuration
+            Returns:
+                None
+            Raises:
+                RunAnsibleModuleFail if ptf test fails
+        """
+
+        # Block voq7
+        dst_dut = get_src_dst_asic_and_duts['dst_dut']
+        dst_asic_index = get_src_dst_asic_and_duts['dst_asic_index']
+        dst_port = dutConfig['dutInterfaces'][dutConfig["testPorts"]["dst_port_id"]]
+        original_pir_voq7 = self.block_queue(dst_dut, dst_port, 7, "voq", dst_asic_index)
+        # Fill leakout of Q7 by ping
+        cmd_opt = "sudo ip netns exec asic{}".format(dst_asic_index)
+        if not dst_dut.sonichost.is_multi_asic:
+            cmd_opt = ""
+        dst_dut.shell("{} ping -I {} -c 50 1.1.1.1 -i 0 -w 0 || true".format(cmd_opt, dst_port))
+
+        # Block oq0
+        original_pir_oq0 = self.block_queue(dst_dut, dst_port, 0, "oq", dst_asic_index)
+
+        testParams = dict()
+        testParams.update(dutTestParams["basicParams"])
+        testParams.update({
+            "dscp": 8,
+            "dst_port_id": dutConfig["testPorts"]["dst_port_id"],
+            "dst_port_ip": dutConfig["testPorts"]["dst_port_ip"],
+            "src_port_id": dutConfig["testPorts"]["src_port_id"],
+            "src_port_ip": dutConfig["testPorts"]["src_port_ip"],
+            "src_port_vlan": dutConfig["testPorts"]["src_port_vlan"],
+            "packet_size": 1350,
+            "pkts_num": PKTS_NUM,
+            "oq_watchdog_enabled": True,
+        })
+
+        self.runPtfTest(
+            ptfhost, testCase="sai_qos_tests.OqWatchdogTest",
+            testParams=testParams)
+
+        # Unblock voq7 and oq0 to restore the system state
+        self.unblock_queue(dst_dut, dst_port, 7, "voq", original_pir_voq7, dst_asic_index)
+        self.unblock_queue(dst_dut, dst_port, 0, "oq", original_pir_oq0, dst_asic_index)
+
+        self.runPtfTest(
+            ptfhost, testCase="sai_qos_tests.TrafficSanityTest",
+            testParams=testParams)

tests/qos/test_voq_watchdog.py

@@ -94,6 +94,10 @@ def testVoqWatchdog(
             ptfhost, testCase="sai_qos_tests.VoqWatchdogTest",
             testParams=testParams)
 
+        self.runPtfTest(
+            ptfhost, testCase="sai_qos_tests.TrafficSanityTest",
+            testParams=testParams)
+
     def testVoqWatchdogDisable(
             self, ptfhost, dutTestParams, dutConfig, dutQosConfig,
             get_src_dst_asic_and_duts, disable_voq_watchdog_function_scope
@@ -129,3 +133,7 @@ def testVoqWatchdogDisable(
         self.runPtfTest(
             ptfhost, testCase="sai_qos_tests.VoqWatchdogTest",
             testParams=testParams)
+
+        self.runPtfTest(
+            ptfhost, testCase="sai_qos_tests.TrafficSanityTest",
+            testParams=testParams)