Enhance qos tests to support single-asic, multi-asic, and multi-dut testing (#8149)

* Enhance qos tests to support single-asic, multi-asic, and multi-dut testing (sandeep: PR#6946)

Cleanup for QoS

Minor fixes for Qos tests

Changes to support QoS multi-asic and ixes to QoS tests for single-asic

 - Ignoring qos/test_buffer.py for T2 and allowing to run test_qos_sai.py on our chassis
 - Changes to support QoS multi-asic

Run only single-asic QoS tests

Integrating cint calls into qos tests

Fixes for QoS multi-asic support

qos masic - docker services should be updated on both src and dst asics

it was being done only on the src asic

More fixes for QoS tests

Adding missing import of 'socket' to sai_qos_base.py file

More fixes to QoS tests based on PR creation

Final fixes for Qos tests with rebase

Fixes for multi-dut and multi-asic in ReleaseAllPorts

Adding missing texttable.py file needed for QoS tests

Fixing typo in QoS tests

Embedding 'show counter' calls in PgSharedWatermark qos test

Adding missing QoS file

Fixing select fixture for QoS test

Fixing the path used for docker-sync-rpc image for QoS tests

qos tuning for LossyQueue parameter

Fixes for PFCXon test

QoS changes after rebase

double commit PR #7109 #7119 #7140 #7154 (#7173)

Changes to DscpToPgmapping and PgSharedWatermarkTest for qos

Updating cint scripts and validating successfully executed

Adding sleep before checking stats for QoS tests

Change PgSharedWatermark_test assert stmt

More fixes for QoS tests

Fixes for QoS QSharedWatermarkTest test

Need to ignore one of the asserts as SAI_INGRESS_PRIORITY_GROUP_STAT_XOFF_ROOM_WATERMARK_BYTES is not supported on DNX

Enabling multi-asic/multi-dut and single-asic mode for QoS tests

All the fixes are in libsai that comes with 202205 - so we can run all
the tests in our weekend pipeline

QoS rebase fixes

Removing ptf_dut_ip - internal Nokia related code

Fixing issue created via rebase

Latest fixes

Fixes to QoS tests for mellanox and cisco-8000 platforms

Fix json.loads exception in dut_qos_maps if corresponding data is not present in the output of sonic-cfggen

Fix to allow tests to run one a single DUT in the testbed that has multiple DUTs defined

Fixed missing 'target' parameter in sai_thrift_read_queue_occupancy calls for cisco-8000

Fixes for Mellanox platforms

fixed indentation error

fixed precheck errors

fixed pre-check errors

fix for tests

* Fixes from 202205 branch

* Fixes for ToR QoS test cases

* Fix for nightly failures

* Fix for nightly failures-2

* dual tor fixes

* dual tor fixes - 2

* dual tor fixes - 3

* pre-check fixes

* pre-check fixes - 2

* pre-check fixes - 3

* pre-check fixes - 4

* pre-check fixes - 5

* pre-check fixes 6

* pre-check fixes 7

---------

Co-authored-by: sanmalho <sandeep.malhotra@nokia.com>

Author: vmittal-msft

tests/common/devices/sonic.py

@@ -1954,22 +1954,6 @@ def is_service_running(self, service_name, docker_name):
 
         return "RUNNING" in service_status
 
-    def remove_ssh_tunnel_sai_rpc(self):
-        """
-        Removes any ssh tunnels if present created for syncd RPC communication
-
-        Returns:
-            None
-        """
-        try:
-            pid_list = self.shell(
-                r'pgrep -f "ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -fN -L \*:9092"'
-            )["stdout_lines"]
-        except RunAnsibleModuleFail:
-            return
-        for pid in pid_list:
-            self.shell("kill {}".format(pid), module_ignore_errors=True)
-
     def get_up_ip_ports(self):
         """
         Get a list for all up ip interfaces

tests/common/devices/sonic_asic.py

@@ -19,6 +19,7 @@ class SonicAsic(object):
 
     _MULTI_ASIC_SERVICE_NAME = "{}@{}"   # service name, asic_id
     _MULTI_ASIC_DOCKER_NAME = "{}{}"     # docker name,  asic_id
+    _RPC_PORT_FOR_SSH_TUNNEL = 9092
 
     def __init__(self, sonichost, asic_index):
         """ Initializing a ASIC on a SONiC host.
@@ -325,6 +326,9 @@ def bgp_drop_rule(self, ip_version, state="present"):
 
         logger.debug(output)
 
+    def get_rpc_port_ssh_tunnel(self):
+        return self._RPC_PORT_FOR_SSH_TUNNEL + self.asic_index
+
     def remove_ssh_tunnel_sai_rpc(self):
         """
         Removes any ssh tunnels if present created for syncd RPC communication
@@ -334,7 +338,15 @@ def remove_ssh_tunnel_sai_rpc(self):
         """
         if not self.sonichost.is_multi_asic:
             return
-        return self.sonichost.remove_ssh_tunnel_sai_rpc()
+
+        try:
+            pid_list = self.sonichost.shell(
+                r'pgrep -f "ssh -o StrictHostKeyChecking=no -fN -L \*:{}"'.format(self.get_rpc_port_ssh_tunnel())
+            )["stdout_lines"]
+        except RunAnsibleModuleFail:
+            return
+        for pid in pid_list:
+            self.shell("kill {}".format(pid), module_ignore_errors=True)
 
     def create_ssh_tunnel_sai_rpc(self):
         """
@@ -362,8 +374,9 @@ def create_ssh_tunnel_sai_rpc(self):
             raise Exception("Invalid V4 address {}".format(ns_docker_if_ipv4))
 
         self.sonichost.shell(
-            ("ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -fN"
-             " -L *:9092:{}:9092 localhost").format(ns_docker_if_ipv4))
+            ("ssh -o StrictHostKeyChecking=no -fN"
+             " -L *:{}:{}:{} localhost").format(self.get_rpc_port_ssh_tunnel(), ns_docker_if_ipv4,
+                                                self._RPC_PORT_FOR_SSH_TUNNEL))
 
     def command(self, cmdstr):
         """

tests/common/fixtures/duthost_utils.py

@@ -448,10 +448,9 @@ def utils_create_test_vlans(duthost, cfg_facts, vlan_ports_list, vlan_intfs_dict
     duthost.shell_cmds(cmds=cmds)
 
 
-@pytest.fixture(scope='module')
-def dut_qos_maps(rand_selected_front_end_dut):
+def _dut_qos_map(dut):
     """
-    A module level fixture to get QoS map from DUT host.
+    A helper function to get QoS map from DUT host.
     Return a dict
     {
         "dscp_to_tc_map": {
@@ -467,43 +466,62 @@ def dut_qos_maps(rand_selected_front_end_dut):
     """
     maps = {}
     try:
-        if rand_selected_front_end_dut.is_multi_asic:
+        if dut.is_multi_asic:
             sonic_cfggen_cmd = "sonic-cfggen -n asic0 -d --var-json"
         else:
             sonic_cfggen_cmd = "sonic-cfggen -d --var-json"
 
         # port_qos_map
-        port_qos_map_data = rand_selected_front_end_dut.shell("{} 'PORT_QOS_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['port_qos_map'] = json.loads(port_qos_map_data) if port_qos_map_data else None
+        port_qos_map = dut.shell("{} 'PORT_QOS_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['port_qos_map'] = json.loads(port_qos_map) if port_qos_map else None
+
         # dscp_to_tc_map
-        dscp_to_tc_map_data = rand_selected_front_end_dut.shell(
-            "{} 'DSCP_TO_TC_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['dscp_to_tc_map'] = json.loads(dscp_to_tc_map_data) if dscp_to_tc_map_data else None
+        dscp_to_tc_map = dut.shell("{} 'DSCP_TO_TC_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['dscp_to_tc_map'] = json.loads(dscp_to_tc_map) if dscp_to_tc_map else None
+
         # tc_to_queue_map
-        tc_to_queue_map_data = rand_selected_front_end_dut.shell(
-            "{} 'TC_TO_QUEUE_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['tc_to_queue_map'] = json.loads(tc_to_queue_map_data) if tc_to_queue_map_data else None
+        tc_to_queue_map = dut.shell("{} 'TC_TO_QUEUE_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['tc_to_queue_map'] = json.loads(tc_to_queue_map) if tc_to_queue_map else None
+
         # tc_to_priority_group_map
-        tc_to_priority_group_map_data = rand_selected_front_end_dut.shell(
-            "{} 'TC_TO_PRIORITY_GROUP_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['tc_to_priority_group_map'] = json.loads(
-            tc_to_priority_group_map_data) if tc_to_priority_group_map_data else None
+        tc_to_priority_group_map = dut.shell("{} 'TC_TO_PRIORITY_GROUP_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['tc_to_priority_group_map'] = json.loads(tc_to_priority_group_map) if tc_to_priority_group_map else None
+
         # tc_to_dscp_map
-        tc_to_dscp_map_data = rand_selected_front_end_dut.shell(
-            "{} 'TC_TO_DSCP_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['tc_to_dscp_map'] = json.loads(tc_to_dscp_map_data) if tc_to_dscp_map_data else None
+        tc_to_dscp_map = dut.shell("{} 'TC_TO_DSCP_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['tc_to_dscp_map'] = json.loads(tc_to_dscp_map) if tc_to_dscp_map else None
     except Exception as e:
         logger.error("Got exception: " + repr(e))
     return maps
 
 
-def separated_dscp_to_tc_map_on_uplink(duthost, dut_qos_maps):
+@pytest.fixture(scope='class')
+def dut_qos_maps(get_src_dst_asic_and_duts):
+    """
+    A class level fixture to get QoS map from DUT host.
+    Return a dict
+    """
+    dut = get_src_dst_asic_and_duts['src_dut']
+    return _dut_qos_map(dut)
+
+
+@pytest.fixture(scope='module')
+def dut_qos_maps_module(rand_selected_front_end_dut):
+    """
+    A module level fixture to get QoS map from DUT host.
+    return a dict
+    """
+    dut = rand_selected_front_end_dut
+    return _dut_qos_map(dut)
+
+
+def separated_dscp_to_tc_map_on_uplink(dut_qos_maps_module):
     """
     A helper function to check if separated DSCP_TO_TC_MAP is applied to
     downlink/unlink ports.
     """
     dscp_to_tc_map_names = set()
-    for port_name, qos_map in list(dut_qos_maps['port_qos_map'].items()):
+    for port_name, qos_map in dut_qos_maps_module['port_qos_map'].items():
         if port_name == "global":
             continue
         dscp_to_tc_map_names.add(qos_map.get("dscp_to_tc_map", ""))
@@ -512,20 +530,20 @@ def separated_dscp_to_tc_map_on_uplink(duthost, dut_qos_maps):
     return False
 
 
-def load_dscp_to_pg_map(duthost, port, dut_qos_maps):
+def load_dscp_to_pg_map(duthost, port, dut_qos_maps_module):
     """
     Helper function to calculate DSCP to PG map for a port.
     The map is derived from DSCP_TO_TC_MAP + TC_TO_PG_MAP
     return a dict like {0:0, 1:1...}
     """
     try:
-        port_qos_map = dut_qos_maps['port_qos_map']
+        port_qos_map = dut_qos_maps_module['port_qos_map']
         dscp_to_tc_map_name = port_qos_map[port]['dscp_to_tc_map'].split('|')[-1].strip(']')
         tc_to_pg_map_name = port_qos_map[port]['tc_to_pg_map'].split('|')[-1].strip(']')
         # Load dscp_to_tc_map
-        dscp_to_tc_map = dut_qos_maps['dscp_to_tc_map'][dscp_to_tc_map_name]
+        dscp_to_tc_map = dut_qos_maps_module['dscp_to_tc_map'][dscp_to_tc_map_name]
         # Load tc_to_pg_map
-        tc_to_pg_map = dut_qos_maps['tc_to_priority_group_map'][tc_to_pg_map_name]
+        tc_to_pg_map = dut_qos_maps_module['tc_to_priority_group_map'][tc_to_pg_map_name]
         # Calculate dscp to pg map
         dscp_to_pg_map = {}
         for dscp, tc in list(dscp_to_tc_map.items()):
@@ -536,20 +554,20 @@ def load_dscp_to_pg_map(duthost, port, dut_qos_maps):
         return {}
 
 
-def load_dscp_to_queue_map(duthost, port, dut_qos_maps):
+def load_dscp_to_queue_map(duthost, port, dut_qos_maps_module):
     """
     Helper function to calculate DSCP to Queue map for a port.
     The map is derived from DSCP_TO_TC_MAP + TC_TO_QUEUE_MAP
     return a dict like {0:0, 1:1...}
     """
     try:
-        port_qos_map = dut_qos_maps['port_qos_map']
+        port_qos_map = dut_qos_maps_module['port_qos_map']
         dscp_to_tc_map_name = port_qos_map[port]['dscp_to_tc_map'].split('|')[-1].strip(']')
         tc_to_queue_map_name = port_qos_map[port]['tc_to_queue_map'].split('|')[-1].strip(']')
         # Load dscp_to_tc_map
-        dscp_to_tc_map = dut_qos_maps['dscp_to_tc_map'][dscp_to_tc_map_name][dscp_to_tc_map_name]
+        dscp_to_tc_map = dut_qos_maps_module['dscp_to_tc_map'][dscp_to_tc_map_name][dscp_to_tc_map_name]
         # Load tc_to_queue_map
-        tc_to_queue_map = dut_qos_maps['tc_to_queue_map'][tc_to_queue_map_name]
+        tc_to_queue_map = dut_qos_maps_module['tc_to_queue_map'][tc_to_queue_map_name]
         # Calculate dscp to queue map
         dscp_to_queue_map = {}
         for dscp, tc in list(dscp_to_tc_map.items()):

tests/common/fixtures/ptfhost_utils.py

@@ -518,28 +518,38 @@ def ptf_test_port_map_active_active(ptfhost, tbinfo, duthosts, mux_server_url, d
                 ]
 
     disabled_ptf_ports = set()
-    for ptf_map in list(tbinfo['topo']['ptf_map_disabled'].values()):
+    for ptf_map in tbinfo['topo']['ptf_map_disabled'].values():
         # Loop ptf_map of each DUT. Each ptf_map maps from ptf port index to dut port index
         disabled_ptf_ports = disabled_ptf_ports.union(set(ptf_map.keys()))
 
-    router_macs = [duthost.facts['router_mac'] for duthost in duthosts]
+    router_macs = []
+    all_dut_names = [duthost.hostname for duthost in duthosts]
+    for a_dut_name in tbinfo['duts']:
+        if a_dut_name in all_dut_names:
+            duthost = duthosts[a_dut_name]
+            router_macs.append(duthost.facts['router_mac'])
+        else:
+            router_macs.append(None)
 
     logger.info('active_dut_map={}'.format(active_dut_map))
     logger.info('disabled_ptf_ports={}'.format(disabled_ptf_ports))
     logger.info('router_macs={}'.format(router_macs))
 
-    asic_idx = 0
     ports_map = {}
     for ptf_port, dut_intf_map in list(tbinfo['topo']['ptf_dut_intf_map'].items()):
         if str(ptf_port) in disabled_ptf_ports:
             # Skip PTF ports that are connected to disabled VLAN interfaces
             continue
+        asic_idx = 0
+        dut_port = None
 
-        if len(list(dut_intf_map.keys())) == 2:
+        if len(dut_intf_map.keys()) == 2:
             # PTF port is mapped to two DUTs -> dualtor topology and the PTF port is a vlan port
             # Packet sent from this ptf port will only be accepted by the active side DUT
-            # DualToR DUTs use same special Vlan interface MAC address
+            # DualToR DUTs use same special Vlan interface MAC addres
             target_dut_indexes = list(map(int, active_dut_map[ptf_port]))
+            target_dut_port = int(list(dut_intf_map.values())[0])
+            target_hostname = duthosts[target_dut_indexes[0]].hostname
             ports_map[ptf_port] = {
                 'target_dut': target_dut_indexes,
                 'target_dest_mac': tbinfo['topo']['properties']['topology']['DUT']['vlan_configs']['one_vlan_a']
@@ -549,12 +559,17 @@ def ptf_test_port_map_active_active(ptfhost, tbinfo, duthosts, mux_server_url, d
             }
         else:
             # PTF port is mapped to single DUT
+            dut_index_for_pft_port = int(list(dut_intf_map.keys())[0])
+            if router_macs[dut_index_for_pft_port] is None:
+                continue
             target_dut_index = int(list(dut_intf_map.keys())[0])
             target_dut_port = int(list(dut_intf_map.values())[0])
             router_mac = router_macs[target_dut_index]
-            dut_port = None
-            if len(duts_minigraph_facts[duthosts[target_dut_index].hostname]) > 1:
-                for list_idx, mg_facts_tuple in enumerate(duts_minigraph_facts[duthosts[target_dut_index].hostname]):
+            target_hostname = tbinfo['duts'][target_dut_index]
+
+            if len(duts_minigraph_facts[target_hostname]) > 1:
+                # Dealing with multi-asic target dut
+                for list_idx, mg_facts_tuple in enumerate(duts_minigraph_facts[target_hostname]):
                     idx, mg_facts = mg_facts_tuple
                     for a_dut_port, a_dut_port_index in list(mg_facts['minigraph_port_indices'].items()):
                         if a_dut_port_index == target_dut_port and "Ethernet-Rec" not in a_dut_port and \
@@ -569,10 +584,18 @@ def ptf_test_port_map_active_active(ptfhost, tbinfo, duthosts, mux_server_url, d
                 'target_dut': [target_dut_index],
                 'target_dest_mac': router_mac,
                 'target_src_mac': [router_mac],
-                'dut_port': dut_port,
                 'asic_idx': asic_idx
             }
 
+        _, asic_mg_facts = duts_minigraph_facts[target_hostname][asic_idx]
+        for a_dut_port, a_dut_port_index in asic_mg_facts['minigraph_port_indices'].items():
+            if a_dut_port_index == target_dut_port and "Ethernet-Rec" not in a_dut_port and \
+                    "Ethernet-IB" not in a_dut_port and "Ethernet-BP" not in a_dut_port:
+                dut_port = a_dut_port
+                break
+
+        ports_map[ptf_port]['dut_port'] = dut_port
+
     logger.debug('ptf_test_port_map={}'.format(json.dumps(ports_map, indent=2)))
 
     ptfhost.copy(content=json.dumps(ports_map), dest=PTF_TEST_PORT_MAP)

tests/conftest.py

@@ -28,6 +28,7 @@
 from tests.common.helpers.parallel import parallel_run
 from tests.common.fixtures.duthost_utils import backup_and_restore_config_db_session    # noqa F401
 from tests.common.fixtures.ptfhost_utils import ptf_portmap_file                        # noqa F401
+from tests.common.fixtures.ptfhost_utils import ptf_test_port_map_active_active         # noqa F401
 from tests.common.fixtures.ptfhost_utils import run_icmp_responder_session              # noqa F401
 
 from tests.common.helpers.constants import (
@@ -1708,6 +1709,59 @@ def duts_running_config_facts(duthosts):
 
 
 @pytest.fixture(scope='class')
+def dut_test_params_qos(duthosts, tbinfo, ptfhost, get_src_dst_asic_and_duts, lower_tor_host, creds,
+                        mux_server_url, mux_status_from_nic_simulator, duts_running_config_facts, duts_minigraph_facts):
+    if 'dualtor' in tbinfo['topo']['name']:
+        all_duts = [lower_tor_host]
+    else:
+        all_duts = get_src_dst_asic_and_duts['all_duts']
+
+    src_asic = get_src_dst_asic_and_duts['src_asic']
+    dst_asic = get_src_dst_asic_and_duts['dst_asic']
+
+    src_dut = get_src_dst_asic_and_duts['src_dut']
+    src_dut_ip = src_dut.host.options['inventory_manager'].get_host(src_dut.hostname).vars['ansible_host']
+    src_server = "{}:{}".format(src_dut_ip, src_asic.get_rpc_port_ssh_tunnel())
+
+    duthost = all_duts[0]
+    mgFacts = duthost.get_extended_minigraph_facts(tbinfo)
+    topo = tbinfo["topo"]["name"]
+
+    rtn_dict = {
+        "topo": topo,
+        "hwsku": mgFacts["minigraph_hwsku"],
+        "basicParams": {
+            "router_mac": duthost.facts["router_mac"],
+            "src_server": src_server,
+            "port_map_file": ptf_test_port_map_active_active(
+                ptfhost, tbinfo, duthosts, mux_server_url,
+                duts_running_config_facts, duts_minigraph_facts,
+                mux_status_from_nic_simulator()),
+            "sonic_asic_type": duthost.facts['asic_type'],
+            "sonic_version": duthost.os_version,
+            "src_dut_index": get_src_dst_asic_and_duts['src_dut_index'],
+            "src_asic_index": get_src_dst_asic_and_duts['src_asic_index'],
+            "dst_dut_index": get_src_dst_asic_and_duts['dst_dut_index'],
+            "dst_asic_index": get_src_dst_asic_and_duts['dst_asic_index'],
+            "dut_username": creds['sonicadmin_user'],
+            "dut_password": creds['sonicadmin_password']
+        },
+
+    }
+
+    # Add dst server info if src and dst asic are different
+    if src_asic != dst_asic:
+        dst_dut = get_src_dst_asic_and_duts['dst_dut']
+        dst_dut_ip = dst_dut.host.options['inventory_manager'].get_host(dst_dut.hostname).vars['ansible_host']
+        rtn_dict["basicParams"]["dst_server"] = "{}:{}".format(dst_dut_ip, dst_asic.get_rpc_port_ssh_tunnel())
+
+    if 'platform_asic' in duthost.facts:
+        rtn_dict['basicParams']["platform_asic"] = duthost.facts['platform_asic']
+
+    yield rtn_dict
+
+
+@ pytest.fixture(scope='class')
 def dut_test_params(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo,
                     ptf_portmap_file, lower_tor_host, creds):   # noqa F811
     """