[Mellanox] [QoS] Support shared headroom pool (#2565)

Author: stephenxs

tests/qos/files/mellanox/qos_param_generator.py

@@ -1,7 +1,7 @@
 import math
 
 class QosParamMellanox(object):
-    def __init__(self, qos_params, asic_type, speed_cable_len, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile, egressLossyProfile):
+    def __init__(self, qos_params, asic_type, speed_cable_len, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile, egressLossyProfile, sharedHeadroomPoolSize):
         asic_param_dic = {
             'spc1': {
                 'cell_size': 96,
@@ -36,6 +36,7 @@ def __init__(self, qos_params, asic_type, speed_cable_len, ingressLosslessProfil
         self.ingressLossyProfile = ingressLossyProfile
         self.egressLosslessProfile = egressLosslessProfile
         self.egressLossyProfile = egressLossyProfile
+        self.sharedHeadroomPoolSize = sharedHeadroomPoolSize
 
         return
 
@@ -57,16 +58,46 @@ def collect_qos_configurations(self):
         xon = int(math.ceil(float(self.ingressLosslessProfile['xon']) / self.cell_size))
         xoff = int(math.ceil(float(self.ingressLosslessProfile['xoff']) / self.cell_size))
         size = int(math.ceil(float(self.ingressLosslessProfile['size']) / self.cell_size))
-        headroom = size
+
+        if self.sharedHeadroomPoolSize:
+            headroom = xon + xoff
+            ingress_lossless_size = int(math.ceil(float(self.ingressLosslessProfile['static_th']) / self.cell_size)) - xon
+        else:
+            headroom = size
+            ingress_lossless_size = int(math.ceil(float(self.ingressLosslessProfile['static_th']) / self.cell_size)) - headroom
         hysteresis = headroom - (xon + xoff)
-        ingress_lossless_size = int(math.ceil(float(self.ingressLosslessProfile['static_th']) / self.cell_size)) - headroom
+
         egress_lossy_size = int(math.ceil(float(self.egressLossyProfile['static_th']) / self.cell_size))
 
         pkts_num_trig_pfc = ingress_lossless_size + xon + hysteresis
-        pkts_num_trig_ingr_drp = ingress_lossless_size + headroom - self.headroom_overhead
+        pkts_num_trig_ingr_drp = ingress_lossless_size + headroom
+        if self.sharedHeadroomPoolSize:
+            pkts_num_trig_ingr_drp += xoff
+        else:
+            pkts_num_trig_ingr_drp -= self.headroom_overhead
         pkts_num_dismiss_pfc = ingress_lossless_size + 1
         pkts_num_trig_egr_drp = egress_lossy_size + 1
 
+        if self.sharedHeadroomPoolSize:
+            ingress_ports_num_shp = 8
+            pkts_num_trig_pfc_shp = []
+            ing_port = 1
+            ingress_ports_list_shp = []
+            occupancy_per_port = ingress_lossless_size
+            for i in range(1, ingress_ports_num_shp):
+                # for the first PG
+                pkts_num_trig_pfc_shp.append(occupancy_per_port + xon + hysteresis)
+                # for the second PG
+                occupancy_per_port /= 2
+                pkts_num_trig_pfc_shp.append(occupancy_per_port + xon + hysteresis)
+                occupancy_per_port /= 2
+                ingress_ports_list_shp.append(ing_port)
+                ing_port += 1
+            self.qos_parameters['pkts_num_trig_pfc_shp'] = pkts_num_trig_pfc_shp
+            self.qos_parameters['src_port_ids'] = ingress_ports_list_shp
+            self.qos_parameters['pkts_num_hdrm_full'] = xoff - 2
+            self.qos_parameters['pkts_num_hdrm_partial'] = xoff - 2
+
         self.qos_parameters['pkts_num_trig_pfc'] = pkts_num_trig_pfc
         self.qos_parameters['pkts_num_trig_ingr_drp'] = pkts_num_trig_ingr_drp
         self.qos_parameters['pkts_num_dismiss_pfc'] = pkts_num_dismiss_pfc
@@ -93,6 +124,18 @@ def calculate_parameters(self):
         pkts_num_trig_egr_drp = self.qos_parameters['pkts_num_trig_egr_drp']
         pkts_num_hysteresis = self.qos_parameters['pkts_num_hysteresis']
 
+        if self.sharedHeadroomPoolSize:
+            hdrm_pool_size = self.qos_params_mlnx[self.speed_cable_len]['hdrm_pool_size']
+            hdrm_pool_size['pkts_num_trig_pfc_shp'] = self.qos_parameters['pkts_num_trig_pfc_shp']
+            hdrm_pool_size['pkts_num_hdrm_full'] = self.qos_parameters['pkts_num_hdrm_full']
+            hdrm_pool_size['pkts_num_hdrm_partial'] = self.qos_parameters['pkts_num_hdrm_partial']
+            hdrm_pool_size['src_port_ids'] = self.qos_parameters['src_port_ids']
+            hdrm_pool_size['pgs_num'] = 2 * len(self.qos_parameters['src_port_ids'])
+            hdrm_pool_size['cell_size'] = self.cell_size
+            hdrm_pool_size['margin'] = 3
+        else:
+            self.qos_params_mlnx[self.speed_cable_len].pop('hdrm_pool_size')    
+
         xoff = {}
         xoff['pkts_num_trig_pfc'] = pkts_num_trig_pfc
         xoff['pkts_num_trig_ingr_drp'] = pkts_num_trig_ingr_drp
@@ -152,3 +195,5 @@ def calculate_parameters(self):
 
         for i in range(4):
             self.qos_params_mlnx['ecn_{}'.format(i+1)]['cell_size'] = self.cell_size
+
+        self.qos_params_mlnx['shared-headroom-pool'] = self.sharedHeadroomPoolSize

tests/qos/files/qos.yml

@@ -40,6 +40,15 @@ qos_params:
                 pkts_num_leak_out: 0
                 pkts_num_fill_min: 0
                 packet_size: 300
+            hdrm_pool_size:
+                dscps: [3, 4]
+                ecn: 1
+                pgs: [3, 4]
+                dst_port_id: 0
+                pkts_num_trig_pfc: 0
+                pkts_num_leak_out: 0
+                pkts_num_fill_min: 0
+                packet_size: 300
         xon_1:
             dscp: 3
             ecn: 1

tests/qos/qos_sai_base.py

@@ -148,6 +148,26 @@ def __getBufferProfile(self, request, duthost, table, port, priorityGroup):
 
         return bufferProfile
 
+    def __getSharedHeadroomPoolSize(self, request, duthost):
+        """
+            Get shared headroom pool size from Redis db
+
+            Args:
+                request (Fixture): pytest request object
+                duthost (AnsibleHost): Device Under Test (DUT)
+
+            Returns:
+                size (str) size of shared headroom pool
+                None if shared headroom pool isn't enabled
+        """
+        result = self.__runRedisCommandOrAssert(
+            duthost,
+            argv = ["redis-cli", "-n", "4", "HGETALL", "BUFFER_POOL|ingress_lossless_pool"]
+        )
+        it = iter(result)
+        ingressLosslessPool = dict(zip(it, it))
+        return ingressLosslessPool.get("xoff")
+
     def __getEcnWredParam(self, duthost, table, port):
         """
             Get ECN/WRED parameters from Redis db
@@ -543,7 +563,7 @@ def disablePacketAging(self, duthosts, rand_one_dut_hostname, stopServices):
             duthost.command("docker exec syncd rm -rf /packets_aging.py")
 
     @pytest.fixture(scope='class', autouse=True)
-    def dutQosConfig(self, duthosts, rand_one_dut_hostname, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile, egressLossyProfile, tbinfo):
+    def dutQosConfig(self, duthosts, rand_one_dut_hostname, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile, egressLossyProfile, sharedHeadroomPoolSize, tbinfo):
         """
             Prepares DUT host QoS configuration
 
@@ -591,11 +611,13 @@ def dutQosConfig(self, duthosts, rand_one_dut_hostname, ingressLosslessProfile,
                 sys.path.append(sub_folder_dir)
             import qos_param_generator
             qpm = qos_param_generator.QosParamMellanox(qosConfigs['qos_params']['mellanox'], dutAsic,
-                                                      portSpeedCableLength,
-                                                      ingressLosslessProfile,
-                                                      ingressLossyProfile,
-                                                      egressLosslessProfile,
-                                                      egressLossyProfile)
+                                                       portSpeedCableLength,
+                                                       ingressLosslessProfile,
+                                                       ingressLossyProfile,
+                                                       egressLosslessProfile,
+                                                       egressLossyProfile,
+                                                       sharedHeadroomPoolSize
+            )
             qosParams = qpm.run()
         else:
             qosParams = qosConfigs['qos_params'][dutAsic]
@@ -687,6 +709,25 @@ def populateArpEntries(self, duthosts, rand_one_dut_hostname, ptfhost, dutTestPa
             testParams.update(dutConfig["testPorts"])
             self.runPtfTest(ptfhost, testCase=saiQosTest, testParams=testParams)
 
+    @pytest.fixture(scope='class', autouse=True)
+    def sharedHeadroomPoolSize(self, request, duthosts, rand_one_dut_hostname):
+        """
+            Retreives shared headroom pool size
+
+            Args:
+                request (Fixture): pytest request object
+                duthost (AnsibleHost): Device Under Test (DUT)
+
+            Returns:
+                size: shared headroom pool size
+                      none if it is not defined
+        """
+        duthost = duthosts[rand_one_dut_hostname]
+        yield self.__getSharedHeadroomPoolSize(
+            request,
+            duthost
+        )
+
     @pytest.fixture(scope='class', autouse=True)
     def ingressLosslessProfile(self, request, duthosts, rand_one_dut_hostname, dutConfig):
         """

tests/qos/test_qos_sai.py

@@ -54,8 +54,9 @@ class TestQosSai(QosSaiBase):
         'Arista-7260CX3-Q64'
     ]
 
-    def testParameter(self, duthost, dutQosConfig, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile):
+    def testParameter(self, duthost, dutConfig, dutQosConfig, ingressLosslessProfile, ingressLossyProfile, egressLosslessProfile):
         logger.info("asictype {}".format(duthost.facts["asic_type"]))
+        logger.info("config {}".format(dutConfig))
         logger.info("qosConfig {}".format(dutQosConfig))
 
     @pytest.mark.parametrize("xoffProfile", ["xoff_1", "xoff_2"])
@@ -168,13 +169,16 @@ def testQosSaiHeadroomPoolSize(self, ptfhost, dutTestParams, dutConfig, dutQosCo
             Raises:
                 RunAnsibleModuleFail if ptf test fails
         """
-        if dutTestParams["hwsku"] not in self.SUPPORTED_HEADROOM_SKUS:
+        if dutTestParams["hwsku"] not in self.SUPPORTED_HEADROOM_SKUS and dutTestParams["basicParams"]["sonic_asic_type"] != "mellanox":
             pytest.skip("Headroom pool size not supported")
 
         portSpeedCableLength = dutQosConfig["portSpeedCableLength"]
         qosConfig = dutQosConfig["param"][portSpeedCableLength]
         testPortIps = dutConfig["testPortIps"]
 
+        if not 'hdrm_pool_size' in qosConfig.keys():
+            pytest.skip("Headroom pool size is not enabled on this DUT")
+
         testParams = dict()
         testParams.update(dutTestParams["basicParams"])
         testParams.update({
@@ -187,11 +191,25 @@ def testQosSaiHeadroomPoolSize(self, ptfhost, dutTestParams, dutConfig, dutQosCo
             "dst_port_id": qosConfig["hdrm_pool_size"]["dst_port_id"],
             "dst_port_ip": testPortIps[qosConfig["hdrm_pool_size"]["dst_port_id"]],
             "pgs_num": qosConfig["hdrm_pool_size"]["pgs_num"],
-            "pkts_num_leak_out": qosConfig["pkts_num_leak_out"],
             "pkts_num_trig_pfc": qosConfig["hdrm_pool_size"]["pkts_num_trig_pfc"],
+            "pkts_num_leak_out": qosConfig["pkts_num_leak_out"],
             "pkts_num_hdrm_full": qosConfig["hdrm_pool_size"]["pkts_num_hdrm_full"],
             "pkts_num_hdrm_partial": qosConfig["hdrm_pool_size"]["pkts_num_hdrm_partial"],
         })
+
+        pkts_num_trig_pfc_shp = qosConfig["hdrm_pool_size"].get("pkts_num_trig_pfc_shp")
+        if pkts_num_trig_pfc_shp:
+            testParams["pkts_num_trig_pfc_shp"] = pkts_num_trig_pfc_shp
+
+        packet_size = qosConfig["hdrm_pool_size"].get("packet_size")
+        if packet_size:
+            testParams["packet_size"] = packet_size
+            testParams["cell_size"] = qosConfig["hdrm_pool_size"]["cell_size"]
+
+        margin = qosConfig["hdrm_pool_size"].get("margin")
+        if margin:
+            testParams["margin"] = margin
+
         self.runPtfTest(ptfhost, testCase="sai_qos_tests.HdrmPoolSizeTest", testParams=testParams)
 
     @pytest.mark.parametrize("bufPool", ["wm_buf_pool_lossless", "wm_buf_pool_lossy"])

tests/saitests/sai_qos_tests.py

@@ -10,6 +10,7 @@
 import sai_base_test
 import operator
 import sys
+import math
 from ptf.testutils import (ptf_ports,
                            simple_arp_packet,
                            send_packet,
@@ -857,10 +858,24 @@ def setUp(self):
         self.pgs_num = self.test_params['pgs_num']
         self.asic_type = self.test_params['sonic_asic_type']
         self.pkts_num_leak_out = self.test_params['pkts_num_leak_out']
-        self.pkts_num_trig_pfc = self.test_params['pkts_num_trig_pfc']
+        self.pkts_num_trig_pfc = self.test_params.get('pkts_num_trig_pfc')
+        if not self.pkts_num_trig_pfc:
+            self.pkts_num_trig_pfc_shp = self.test_params.get('pkts_num_trig_pfc_shp')
         self.pkts_num_hdrm_full = self.test_params['pkts_num_hdrm_full']
         self.pkts_num_hdrm_partial = self.test_params['pkts_num_hdrm_partial']
-        print >> sys.stderr, ("pkts num: leak_out: %d, trig_pfc: %d, hdrm_full: %d, hdrm_partial: %d" % (self.pkts_num_leak_out, self.pkts_num_trig_pfc, self.pkts_num_hdrm_full, self.pkts_num_hdrm_partial))
+        packet_size = self.test_params.get('packet_size')
+        if packet_size:
+            self.pkt_size = packet_size
+            cell_size = self.test_params.get('cell_size')
+            self.pkt_size_factor = int(math.ceil(float(packet_size)/cell_size))
+        else:
+            self.pkt_size = 64
+            self.pkt_size_factor = 1
+
+        if self.pkts_num_trig_pfc:
+            print >> sys.stderr, ("pkts num: leak_out: %d, trig_pfc: %d, hdrm_full: %d, hdrm_partial: %d, pkt_size %d" % (self.pkts_num_leak_out, self.pkts_num_trig_pfc, self.pkts_num_hdrm_full, self.pkts_num_hdrm_partial, self.pkt_size))
+        elif self.pkts_num_trig_pfc_shp:
+            print >> sys.stderr, ("pkts num: leak_out: {}, trig_pfc: {}, hdrm_full: {}, hdrm_partial: {}, pkt_size {}".format(self.pkts_num_leak_out, self.pkts_num_trig_pfc_shp, self.pkts_num_hdrm_full, self.pkts_num_hdrm_partial, self.pkt_size))            
         sys.stderr.flush()
 
         self.dst_port_mac = self.dataplane.get_mac(0, self.dst_port_id)
@@ -894,7 +909,9 @@ def tearDown(self):
         sai_base_test.ThriftInterfaceDataPlane.tearDown(self)
 
     def runTest(self):
-        margin = 0
+        margin = self.test_params.get('margin')
+        if not margin:
+            margin = 0
         sidx_dscp_pg_tuples = [(sidx, dscp, self.pgs[pgidx]) for sidx, sid in enumerate(self.src_port_ids) for pgidx, dscp in enumerate(self.dscps)]
         assert(len(sidx_dscp_pg_tuples) >= self.pgs_num)
         print >> sys.stderr, sidx_dscp_pg_tuples
@@ -911,7 +928,7 @@ def runTest(self):
         try:
             # send packets to leak out
             sidx = 0
-            pkt = simple_tcp_packet(pktlen=64,
+            pkt = simple_tcp_packet(pktlen=self.pkt_size,
                         eth_dst=self.router_mac if self.router_mac != '' else self.dst_port_mac,
                         eth_src=self.src_port_macs[sidx],
                         ip_src=self.src_port_ips[sidx],
@@ -926,15 +943,19 @@ def runTest(self):
                 tos = sidx_dscp_pg_tuples[i][1] << 2
                 tos |= self.ecn
                 ttl = 64
-                default_packet_length = 64
+                default_packet_length = self.pkt_size
                 pkt = simple_tcp_packet(pktlen=default_packet_length,
                                         eth_dst=self.router_mac if self.router_mac != '' else self.dst_port_mac,
                                         eth_src=self.src_port_macs[sidx_dscp_pg_tuples[i][0]],
                                         ip_src=self.src_port_ips[sidx_dscp_pg_tuples[i][0]],
                                         ip_dst=self.dst_port_ip,
                                         ip_tos=tos,
                                         ip_ttl=ttl)
-                send_packet(self, self.src_port_ids[sidx_dscp_pg_tuples[i][0]], pkt, self.pkts_num_trig_pfc)
+                if self.pkts_num_trig_pfc:
+                    pkts_num_trig_pfc = self.pkts_num_trig_pfc
+                else:
+                    pkts_num_trig_pfc = self.pkts_num_trig_pfc_shp[i]
+                send_packet(self, self.src_port_ids[sidx_dscp_pg_tuples[i][0]], pkt, pkts_num_trig_pfc / self.pkt_size_factor)
 
             print >> sys.stderr, "Service pool almost filled"
             sys.stderr.flush()
@@ -946,7 +967,7 @@ def runTest(self):
                 tos = sidx_dscp_pg_tuples[i][1] << 2
                 tos |= self.ecn
                 ttl = 64
-                default_packet_length = 64
+                default_packet_length = self.pkt_size
                 pkt = simple_tcp_packet(pktlen=default_packet_length,
                                         eth_dst=self.router_mac if self.router_mac != '' else self.dst_port_mac,
                                         eth_src=self.src_port_macs[sidx_dscp_pg_tuples[i][0]],
@@ -982,7 +1003,7 @@ def runTest(self):
                 tos = sidx_dscp_pg_tuples[i][1] << 2
                 tos |= self.ecn
                 ttl = 64
-                default_packet_length = 64
+                default_packet_length = self.pkt_size
                 pkt = simple_tcp_packet(pktlen=default_packet_length,
                                         eth_dst=self.router_mac if self.router_mac != '' else self.dst_port_mac,
                                         eth_src=self.src_port_macs[sidx_dscp_pg_tuples[i][0]],
@@ -991,7 +1012,7 @@ def runTest(self):
                                         ip_tos=tos,
                                         ip_ttl=ttl)
 
-                send_packet(self, self.src_port_ids[sidx_dscp_pg_tuples[i][0]], pkt, self.pkts_num_hdrm_full if i != self.pgs_num - 1 else self.pkts_num_hdrm_partial)
+                send_packet(self, self.src_port_ids[sidx_dscp_pg_tuples[i][0]], pkt, self.pkts_num_hdrm_full / self.pkt_size_factor if i != self.pgs_num - 1 else self.pkts_num_hdrm_partial / self.pkt_size_factor)
                 # allow enough time for the dut to sync up the counter values in counters_db
                 time.sleep(8)