Enhance qos tests to support single-asic, multi-asic, and multi-dut testing

tests/common/devices/sonic.py

@@ -1873,22 +1873,6 @@ def is_service_running(self, service_name, docker_name):
 
         return "RUNNING" in service_status
 
-    def remove_ssh_tunnel_sai_rpc(self):
-        """
-        Removes any ssh tunnels if present created for syncd RPC communication
-
-        Returns:
-            None
-        """
-        try:
-            pid_list = self.shell(
-                'pgrep -f "ssh -o StrictHostKeyChecking=no -fN -L \*:9092"'
-            )["stdout_lines"]
-        except RunAnsibleModuleFail:
-            return
-        for pid in pid_list:
-            self.shell("kill {}".format(pid), module_ignore_errors=True)
-
     def get_up_ip_ports(self):
         """
         Get a list for all up ip interfaces

tests/common/devices/sonic_asic.py

@@ -18,6 +18,7 @@ class SonicAsic(object):
 
     _MULTI_ASIC_SERVICE_NAME = "{}@{}"   # service name, asic_id
     _MULTI_ASIC_DOCKER_NAME = "{}{}"     # docker name,  asic_id
+    _RPC_PORT_FOR_SSH_TUNNEL = 9092
 
     def __init__(self, sonichost, asic_index):
         """ Initializing a ASIC on a SONiC host.
@@ -325,6 +326,9 @@ def bgp_drop_rule(self, ip_version, state="present"):
 
         logger.debug(output)
 
+    def get_rpc_port_ssh_tunnel(self):
+        return self._RPC_PORT_FOR_SSH_TUNNEL + self.asic_index
+
     def remove_ssh_tunnel_sai_rpc(self):
         """
         Removes any ssh tunnels if present created for syncd RPC communication
@@ -334,7 +338,15 @@ def remove_ssh_tunnel_sai_rpc(self):
         """
         if not self.sonichost.is_multi_asic:
             return
-        return self.sonichost.remove_ssh_tunnel_sai_rpc()
+
+        try:
+            pid_list = self.sonichost.shell(
+                r'pgrep -f "ssh -o StrictHostKeyChecking=no -fN -L \*:{}"'.format(self.get_rpc_port_ssh_tunnel())
+            )["stdout_lines"]
+        except RunAnsibleModuleFail:
+            return
+        for pid in pid_list:
+            self.shell("kill {}".format(pid), module_ignore_errors=True)
 
     def create_ssh_tunnel_sai_rpc(self):
         """
@@ -363,9 +375,9 @@ def create_ssh_tunnel_sai_rpc(self):
 
         self.sonichost.shell(
             ("ssh -o StrictHostKeyChecking=no -fN"
-             " -L *:9092:{}:9092 localhost"
-            ).format(ns_docker_if_ipv4)
-        )
+             " -L *:{}:{}:{} localhost").format(self.get_rpc_port_ssh_tunnel(), ns_docker_if_ipv4,
+                                                self._RPC_PORT_FOR_SSH_TUNNEL ))
+
 
     def command(self, cmdstr):
         """

tests/common/fixtures/duthost_utils.py

@@ -434,8 +434,8 @@ def utils_create_test_vlans(duthost, cfg_facts, vlan_ports_list, vlan_intfs_dict
     duthost.shell_cmds(cmds=cmds)
 
 
-@pytest.fixture(scope='module')
-def dut_qos_maps(rand_selected_front_end_dut):
+@pytest.fixture(scope='class')
+def dut_qos_maps(get_src_dst_asic_and_duts):
     """
     A module level fixture to get QoS map from DUT host.
     Return a dict
@@ -452,32 +452,32 @@ def dut_qos_maps(rand_selected_front_end_dut):
     or an empty dict if failed to parse the output
     """
     maps = {}
+    dut = get_src_dst_asic_and_duts['src_dut']
     try:
-        if rand_selected_front_end_dut.is_multi_asic:
+        if dut.is_multi_asic:
             sonic_cfggen_cmd = "sonic-cfggen -n asic0 -d --var-json"
         else:
             sonic_cfggen_cmd = "sonic-cfggen -d --var-json"
 
         # port_qos_map
-        port_qos_map_data = rand_selected_front_end_dut.shell("{} 'PORT_QOS_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['port_qos_map'] = json.loads(port_qos_map_data) if port_qos_map_data else None
+        port_qos_map = dut.shell("{} 'PORT_QOS_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['port_qos_map'] = json.loads(port_qos_map) if port_qos_map else None
+
         # dscp_to_tc_map
-        dscp_to_tc_map_data = rand_selected_front_end_dut.shell(
-            "{} 'DSCP_TO_TC_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['dscp_to_tc_map'] = json.loads(dscp_to_tc_map_data) if dscp_to_tc_map_data else None
+        dscp_to_tc_map = dut.shell("{} 'DSCP_TO_TC_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['dscp_to_tc_map'] = json.loads(dscp_to_tc_map) if dscp_to_tc_map else None
+
         # tc_to_queue_map
-        tc_to_queue_map_data = rand_selected_front_end_dut.shell(
-            "{} 'TC_TO_QUEUE_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['tc_to_queue_map'] = json.loads(tc_to_queue_map_data) if tc_to_queue_map_data else None
+        tc_to_queue_map = dut.shell("{} 'TC_TO_QUEUE_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['tc_to_queue_map'] = json.loads(tc_to_queue_map) if tc_to_queue_map else None
+
         # tc_to_priority_group_map
-        tc_to_priority_group_map_data = rand_selected_front_end_dut.shell(
-            "{} 'TC_TO_PRIORITY_GROUP_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['tc_to_priority_group_map'] = json.loads(
-            tc_to_priority_group_map_data) if tc_to_priority_group_map_data else None
+        tc_to_priority_group_map = dut.shell("{} 'TC_TO_PRIORITY_GROUP_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['tc_to_priority_group_map'] = json.loads(tc_to_priority_group_map) if tc_to_priority_group_map else None
+
         # tc_to_dscp_map
-        tc_to_dscp_map_data = rand_selected_front_end_dut.shell(
-            "{} 'TC_TO_DSCP_MAP'".format(sonic_cfggen_cmd))['stdout']
-        maps['tc_to_dscp_map'] = json.loads(tc_to_dscp_map_data) if tc_to_dscp_map_data else None
+        tc_to_dscp_map = dut.shell("{} 'TC_TO_DSCP_MAP'".format(sonic_cfggen_cmd))['stdout']
+        maps['tc_to_dscp_map'] = json.loads(tc_to_dscp_map) if tc_to_dscp_map else None
     except Exception as e:
         logger.error("Got exception: " + repr(e))
     return maps

tests/common/fixtures/ptfhost_utils.py

@@ -508,24 +508,34 @@ def ptf_test_port_map_active_active(ptfhost, tbinfo, duthosts, mux_server_url, d
         # Loop ptf_map of each DUT. Each ptf_map maps from ptf port index to dut port index
         disabled_ptf_ports = disabled_ptf_ports.union(set(ptf_map.keys()))
 
-    router_macs = [duthost.facts['router_mac'] for duthost in duthosts]
+    router_macs = []
+    all_dut_names = [duthost.hostname for duthost in duthosts]
+    for a_dut_name in tbinfo['duts']:
+        if a_dut_name in all_dut_names:
+            duthost = duthosts[a_dut_name]
+            router_macs.append(duthost.facts['router_mac'])
+        else:
+            router_macs.append(None)
 
     logger.info('active_dut_map={}'.format(active_dut_map))
     logger.info('disabled_ptf_ports={}'.format(disabled_ptf_ports))
     logger.info('router_macs={}'.format(router_macs))
 
-    asic_idx = 0
     ports_map = {}
     for ptf_port, dut_intf_map in tbinfo['topo']['ptf_dut_intf_map'].items():
         if str(ptf_port) in disabled_ptf_ports:
             # Skip PTF ports that are connected to disabled VLAN interfaces
             continue
+        asic_idx = 0
+        dut_port = None
 
         if len(dut_intf_map.keys()) == 2:
             # PTF port is mapped to two DUTs -> dualtor topology and the PTF port is a vlan port
             # Packet sent from this ptf port will only be accepted by the active side DUT
-            # DualToR DUTs use same special Vlan interface MAC address
+            # DualToR DUTs use same special Vlan interface MAC addres
             target_dut_indexes = list(map(int, active_dut_map[ptf_port]))
+            target_dut_port = int(list(dut_intf_map.values())[0])
+            target_hostname = duthosts[target_dut_indexes[0]].hostname
             ports_map[ptf_port] = {
                 'target_dut': target_dut_indexes,
                 'target_dest_mac': tbinfo['topo']['properties']['topology']['DUT']['vlan_configs']['one_vlan_a']
@@ -535,30 +545,39 @@ def ptf_test_port_map_active_active(ptfhost, tbinfo, duthosts, mux_server_url, d
             }
         else:
             # PTF port is mapped to single DUT
+            dut_index_for_pft_port = int(dut_intf_map.keys()[0])
+            if router_macs[dut_index_for_pft_port] is None:
+                continue
             target_dut_index = int(list(dut_intf_map.keys())[0])
             target_dut_port = int(list(dut_intf_map.values())[0])
             router_mac = router_macs[target_dut_index]
-            dut_port = None
-            if len(duts_minigraph_facts[duthosts[target_dut_index].hostname]) > 1:
-                for list_idx, mg_facts_tuple in enumerate(duts_minigraph_facts[duthosts[target_dut_index].hostname]):
+            target_hostname = tbinfo['duts'][target_dut_index]
+
+            if len(duts_minigraph_facts[target_hostname]) > 1:
+                # Dealing with multi-asic target dut
+                for list_idx, mg_facts_tuple in enumerate(duts_minigraph_facts[target_hostname]):
                     idx, mg_facts = mg_facts_tuple
                     if target_dut_port in list(mg_facts['minigraph_port_indices'].values()):
-                        router_mac = duts_running_config_facts[duthosts[target_dut_index].hostname][list_idx][1]\
+                        router_mac = duts_running_config_facts[target_hostname][list_idx][1]\
                         ['DEVICE_METADATA']['localhost']['mac'].lower()
                         asic_idx = idx
-                        for a_dut_port, a_dut_port_index in mg_facts['minigraph_port_indices'].items():
-                            if a_dut_port_index == target_dut_port and "Ethernet-Rec" not in a_dut_port and \
-                                    "Ethernet-IB" not in a_dut_port and "Ethernet-BP" not in a_dut_port:
-                                dut_port = a_dut_port
-                        break
+
             ports_map[ptf_port] = {
                 'target_dut': [target_dut_index],
                 'target_dest_mac': router_mac,
                 'target_src_mac': [router_mac],
-                'dut_port': dut_port,
                 'asic_idx': asic_idx
             }
 
+        _, asic_mg_facts = duts_minigraph_facts[target_hostname][asic_idx]
+        for a_dut_port, a_dut_port_index in asic_mg_facts['minigraph_port_indices'].items():
+            if a_dut_port_index == target_dut_port and "Ethernet-Rec" not in a_dut_port and \
+                    "Ethernet-IB" not in a_dut_port and "Ethernet-BP" not in a_dut_port:
+                dut_port = a_dut_port
+                break
+
+        ports_map[ptf_port]['dut_port'] = dut_port
+
     logger.debug('ptf_test_port_map={}'.format(json.dumps(ports_map, indent=2)))
 
     ptfhost.copy(content=json.dumps(ports_map), dest=PTF_TEST_PORT_MAP)

tests/conftest.py

@@ -28,6 +28,7 @@
 from tests.common.fixtures.duthost_utils import backup_and_restore_config_db_session
 from tests.common.fixtures.ptfhost_utils import ptf_portmap_file                # lgtm[py/unused-import]
 from tests.common.fixtures.ptfhost_utils import run_icmp_responder_session      # lgtm[py/unused-import]
+from tests.common.fixtures.ptfhost_utils import ptf_test_port_map_active_active         # noqa F401
 
 from tests.common.helpers.constants import (
     ASIC_PARAM_TYPE_ALL, ASIC_PARAM_TYPE_FRONTEND, DEFAULT_ASIC_ID, ASICS_PRESENT
@@ -1599,6 +1600,59 @@ def duts_running_config_facts(duthosts):
     return cfg_facts
 
 @pytest.fixture(scope='class')
+def dut_test_params_qos(duthosts, tbinfo, ptfhost, get_src_dst_asic_and_duts, lower_tor_host, creds,
+                        mux_server_url, mux_status_from_nic_simulator, duts_running_config_facts, duts_minigraph_facts):
+    if 'dualtor' in tbinfo['topo']['name']:
+        all_duts = [lower_tor_host]
+    else:
+        all_duts = get_src_dst_asic_and_duts['all_duts']
+
+    src_asic = get_src_dst_asic_and_duts['src_asic']
+    dst_asic = get_src_dst_asic_and_duts['dst_asic']
+
+    src_dut = get_src_dst_asic_and_duts['src_dut']
+    src_dut_ip = src_dut.host.options['inventory_manager'].get_host(src_dut.hostname).vars['ansible_host']
+    src_server = "{}:{}".format(src_dut_ip, src_asic.get_rpc_port_ssh_tunnel())
+
+    duthost = all_duts[0]
+    mgFacts = duthost.get_extended_minigraph_facts(tbinfo)
+    topo = tbinfo["topo"]["name"]
+
+    rtn_dict = {
+        "topo": topo,
+        "hwsku": mgFacts["minigraph_hwsku"],
+        "basicParams": {
+            "router_mac": duthost.facts["router_mac"],
+            "src_server" : src_server,
+            "port_map_file": ptf_test_port_map_active_active(
+                ptfhost, tbinfo, duthosts, mux_server_url,
+                duts_running_config_facts, duts_minigraph_facts,
+                mux_status_from_nic_simulator()),
+            "sonic_asic_type": duthost.facts['asic_type'],
+            "sonic_version": duthost.os_version,
+            "src_dut_index": get_src_dst_asic_and_duts['src_dut_index'],
+            "src_asic_index": get_src_dst_asic_and_duts['src_asic_index'],
+            "dst_dut_index": get_src_dst_asic_and_duts['dst_dut_index'],
+            "dst_asic_index": get_src_dst_asic_and_duts['dst_asic_index'],
+            "dut_username": creds['sonicadmin_user'],
+            "dut_password": creds['sonicadmin_password']
+        },
+
+    }
+
+    # Add dst server info if src and dst asic are different
+    if src_asic != dst_asic:
+        dst_dut = get_src_dst_asic_and_duts['dst_dut']
+        dst_dut_ip = dst_dut.host.options['inventory_manager'].get_host(dst_dut.hostname).vars['ansible_host']
+        rtn_dict["basicParams"]["dst_server"] = "{}:{}".format(dst_dut_ip, dst_asic.get_rpc_port_ssh_tunnel())
+
+    if 'platform_asic' in duthost.facts:
+        rtn_dict['basicParams']["platform_asic"] = duthost.facts['platform_asic']
+
+    yield rtn_dict
+
+
+@ pytest.fixture(scope='class')
 def dut_test_params(duthosts, enum_rand_one_per_hwsku_frontend_hostname, tbinfo,
                     ptf_portmap_file, lower_tor_host, creds):   # noqa F811
     """

tests/qos/files/mellanox/qos_param_generator.py

@@ -1,7 +1,9 @@
 import math
 
 class QosParamMellanox(object):
-    def __init__(self, qos_params, asic_type, speed_cable_len, dutConfig, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile, egressLossyProfile, sharedHeadroomPoolSize, dualTor):
+    def __init__(self, qos_params, asic_type, speed_cable_len, dutConfig, ingressLosslessProfile,
+                 ingressLossyProfile, egressLosslessProfile, egressLossyProfile, sharedHeadroomPoolSize,
+                 dualTor, src_dut_index, src_asic_index, dst_asic_index, dst_dut_index):
         self.asic_param_dic = {
             'spc1': {
                 'cell_size': 96,
@@ -44,7 +46,10 @@ def __init__(self, qos_params, asic_type, speed_cable_len, dutConfig, ingressLos
             self.sharedHeadroomPoolSize = None
         self.dutConfig = dutConfig
         self.dualTor = dualTor
-
+        self.src_dut_index = src_dut_index
+        self.src_asic_index = src_asic_index
+        self.dst_dut_index = dst_dut_index
+        self.dst_asic_index = dst_asic_index
         return
 
     def run(self):
@@ -88,18 +93,19 @@ def collect_qos_configurations(self):
         pkts_num_trig_egr_drp = egress_lossy_size + 1
 
         if self.sharedHeadroomPoolSize:
-            testPortIds = self.dutConfig['testPortIds']
+            src_testPortIds = self.dutConfig['testPortIds'][self.src_dut_index][self.src_asic_index]
+            dst_testPortIds = self.dutConfig['testPortIds'][self.dst_dut_index][self.dst_asic_index]
             ingress_ports_num_shp = 8
             pkts_num_trig_pfc_shp = []
             ingress_ports_list_shp = []
             occupancy_per_port = ingress_lossless_size
-            self.qos_parameters['dst_port_id'] = testPortIds[0]
+            self.qos_parameters['dst_port_id'] = dst_testPortIds[0]
             pgs_per_port = 2 if not self.dualTor else 4
             for i in range(1, ingress_ports_num_shp):
                 for j in range(pgs_per_port):
                     pkts_num_trig_pfc_shp.append(occupancy_per_port + xon + hysteresis)
                     occupancy_per_port /= 2
-                ingress_ports_list_shp.append(testPortIds[i])
+                ingress_ports_list_shp.append(src_testPortIds[i])
             self.qos_parameters['pkts_num_trig_pfc_shp'] = pkts_num_trig_pfc_shp
             self.qos_parameters['src_port_ids'] = ingress_ports_list_shp
             self.qos_parameters['pkts_num_hdrm_full'] = xoff - 2
@@ -209,4 +215,4 @@ def calculate_parameters(self):
             self.qos_params_mlnx['ecn_{}'.format(i+1)]['cell_size'] = self.cell_size
 
         self.qos_params_mlnx['shared-headroom-pool'] = self.sharedHeadroomPoolSize
-        self.qos_params_mlnx['pkts_num_private_headrooom'] = self.asic_param_dic[self.asic_type]['private_headroom']
+        self.qos_params_mlnx['pkts_num_private_headrooom'] = self.asic_param_dic[self.asic_type]['private_headroom']

tests/qos/qos_helpers.py

@@ -35,6 +35,8 @@ def eos_to_linux_intf(eos_intf_name, hwsku=None):
     """
     if hwsku == "MLNX-OS":
         linux_intf_name = eos_intf_name.replace("ernet 1/", "sl1p").replace("/", "sp")
+    elif "Nokia" in hwsku:
+        linux_intf_name = eos_intf_name
     else:
         linux_intf_name = eos_intf_name.replace('Ethernet', 'et').replace('/', '_')
     return linux_intf_name