[mmu probing] pr09.test: Add production probe test and infrastructure updates

tests/common/plugins/conditional_mark/tests_mark_conditions.yaml

@@ -3995,6 +3995,30 @@ qos/test_qos_masic.py:
       - "topo_type in ['m0', 'mx', 'm1']"
       - "asic_type in ['vs']"
 
+qos/test_qos_probe.py:
+  skip:
+    reason: "Buffer threshold probing tests need platform-specific validation. Tracked by per-ASIC GitHub issues."
+    conditions_logical_operator: or
+    conditions:
+      # Virtual Switch support
+      - "asic_type in ['vs'] and https://github.com/sonic-net/sonic-mgmt/issues/22599"
+      # Cisco-8000 GB series (8102/8101 GB SKUs)
+      - "hwsku in ['Cisco-8102-C64', 'Cisco-8101-O8C48', 'Cisco-8102-28FH-DPU-O'] and https://github.com/sonic-net/sonic-mgmt/issues/22601"
+      # Cisco-8000 GR/GR2 series (8101 GR/GR2 SKUs)
+      - "hwsku in ['Cisco-8101-O32', 'Cisco-8101-O8V48'] and https://github.com/sonic-net/sonic-mgmt/issues/22602"
+      # Broadcom TD3 (Trident 3)
+      - "hwsku in ['Arista-7050CX3-32C-C32', 'Arista-7050CX3-32S-C32'] and https://github.com/sonic-net/sonic-mgmt/issues/22603"
+      # Broadcom TH (Tomahawk)
+      - "hwsku in ['Arista-7060CX-32S-C32', 'Arista-7060CX-32S-Q32'] and https://github.com/sonic-net/sonic-mgmt/issues/22604"
+      # Broadcom TH2 (Tomahawk 2)
+      - "hwsku in ['Arista-7260CX3-C64', 'Arista-7260CX3-D108C8', 'Arista-7260CX3-D108C10'] and https://github.com/sonic-net/sonic-mgmt/issues/22605"
+      # Broadcom TH5 (Tomahawk 5)
+      - "hwsku in ['Arista-7060X6-16PE-384C-B-O128S2', 'Arista-7060X6-64PE-B-O128'] and https://github.com/sonic-net/sonic-mgmt/issues/22606"
+      # Mellanox SPC1 (Spectrum 1)
+      - "hwsku in ['Mellanox-SN2700'] and https://github.com/sonic-net/sonic-mgmt/issues/22607"
+      # Mellanox SPC3 (Spectrum 3)
+      - "hwsku in ['Mellanox-SN4600C-C64'] and https://github.com/sonic-net/sonic-mgmt/issues/22608"
+
 qos/test_qos_sai.py:
   skip:
     reason: "qos_sai tests not supported on t1 topo / M* topo does not support qos and It is skipped for '202412' for now"

tests/conftest.py

@@ -236,6 +236,10 @@ def pytest_addoption(parser):
     parser.addoption("--py_saithrift_url", action="store", default=None, type=str,
                      help="Specify the url of the saithrift package to be installed on the ptf "
                           "(should be http://<serverip>/path/python-saithrift_0.9.4_amd64.deb")
+    parser.addoption("--enable_qos_ptf_pdb", action="store_true", default=False,
+                     help="Enable QoS PTF test debugging mode with pdb breakpoint")
+    parser.addoption("--ingress_drop_probing", action="store_true", default=False,
+                     help="Enable ingress drop threshold probing instead of PFC xoff probing")
 
     #########################
     #   post-test options   #

tests/ptf_runner.py

@@ -83,14 +83,19 @@ def get_test_path(testdir, testname):
     """
     Returns two values
     - first: the complete path of the test based on testdir and testname.
-    - second: True if file is in 'py3' False otherwise
+    - second: True if file is in a subdirectory ('py3' or 'probe'), False otherwise.
+              Originally only 'py3' existed; 'probe' was added for the MMU threshold
+              probing framework.
     Raises FileNotFoundError if file is not found
     """
     curr_path = os.path.dirname(os.path.abspath(__file__))
     base_path = pathlib.Path(curr_path).joinpath('..').joinpath('ansible/roles/test/files').joinpath(testdir)
     idx = testname.find('.')
     test_fname = testname + '.py' if idx == -1 else testname[:idx] + '.py'
     chk_path = base_path.joinpath('py3').joinpath(test_fname)
+    if chk_path.exists():
+        return chk_path, True
+    chk_path = base_path.joinpath('probe').joinpath(test_fname)
     if chk_path.exists():
         return chk_path, True
     chk_path = base_path.joinpath(test_fname)
@@ -122,7 +127,8 @@ def ptf_runner(host, testdir, testname, platform_dir=None, params={},
                socket_recv_size=None, log_file=None,
                ptf_collect_dir="./logs/ptf_collect/",
                device_sockets=[], timeout=0, custom_options="",
-               module_ignore_errors=False, is_python3=None, async_mode=False, pdb=False):
+               module_ignore_errors=False, is_python3=None, async_mode=False, pdb=False,
+               test_subdir='py3'):
     dut_type = get_dut_type(host)
     asic_type = get_asic_type(host)
     kvm_support = params.get("kvm_support", False)
@@ -133,8 +139,8 @@ def ptf_runner(host, testdir, testname, platform_dir=None, params={},
     cmd = ""
     ptf_img_type = get_ptf_image_type(host)
     logger.info('PTF image type: {}'.format(ptf_img_type))
-    test_fpath, in_py3 = get_test_path(testdir, testname)
-    logger.info('Test file path {}, in py3: {}'.format(test_fpath, in_py3))
+    test_fpath, in_subdir = get_test_path(testdir, testname)
+    logger.info('Test file path {}, in subdir: {}'.format(test_fpath, in_subdir))
     is_python3 = is_py3_compat(test_fpath)
 
     # The logic below automatically chooses the PTF binary to execute a test script
@@ -161,8 +167,8 @@ def ptf_runner(host, testdir, testname, platform_dir=None, params={},
             err_msg = 'cannot run Python 2 test in a Python 3 only {} {}'.format(testdir, testname)
             raise Exception(err_msg)
 
-    if in_py3:
-        tdir = pathlib.Path(testdir).joinpath('py3')
+    if in_subdir:
+        tdir = pathlib.Path(testdir).joinpath(test_subdir)
         cmd = "{} --test-dir {} {}".format(ptf_cmd, tdir, testname)
     else:
         cmd = "{} --test-dir {} {}".format(ptf_cmd, testdir, testname)

tests/qos/qos_sai_base.py

@@ -145,7 +145,8 @@ def dutTestParams(self, duthosts, dut_test_params_qos, tbinfo, get_src_dst_asic_
 
         yield dut_test_params_qos
 
-    def runPtfTest(self, ptfhost, testCase='', testParams={}, relax=False, pdb=False, skip_pcap=False):
+    def runPtfTest(self, ptfhost, testCase='', testParams={}, relax=False, pdb=False,
+                   skip_pcap=False, test_subdir='py3'):
         """
             Runs QoS SAI test case on PTF host
 
@@ -186,7 +187,8 @@ def runPtfTest(self, ptfhost, testCase='', testParams={}, relax=False, pdb=False
             timeout=1850,
             socket_recv_size=16384,
             custom_options=custom_options,
-            pdb=pdb
+            pdb=pdb,
+            test_subdir=test_subdir
         )
 
 
@@ -635,6 +637,97 @@ def __assignTestPortIps(self, mgFacts, topo, lower_tor_host):  # noqa: F811
 
         return dutPortIps
 
+    def replaceNonExistentPortId(self, availablePortIds, portIds):
+        '''
+        if port id of availablePortIds/dst_port_ids is not existing in availablePortIds
+        replace it with correct one, make sure all port id is valid
+        e.g.
+            Given below parameter:
+                availablePortIds: [0, 2, 4, 6, 8, 10, 16, 18, 20, 22, 24, 26,
+                                   28, 30, 32, 34, 36, 38, 44, 46, 48, 50, 52, 54]
+                portIds: [1, 2, 3, 4, 5, 6, 7, 8, 9]
+            get result:
+                portIds: [0, 2, 16, 4, 18, 6, 20, 8, 22]
+        '''
+        if len(portIds) > len(availablePortIds):
+            logger.info('no enough ports for test')
+            return False
+
+        # cache available as free port pool
+        freePorts = [pid for pid in availablePortIds]
+
+        # record invaild port
+        # and remove valid port from free port pool
+        invalid = []
+        for idx, pid in enumerate(portIds):
+            if pid not in freePorts:
+                invalid.append(idx)
+            else:
+                freePorts.remove(pid)
+
+        # replace invalid port from free port pool
+        for idx in invalid:
+            portIds[idx] = freePorts.pop(0)
+
+        return True
+
+    def updateTestPortIdIp(self, dutConfig, get_src_dst_asic_and_duts, qosParams=None):
+        src_dut_index = get_src_dst_asic_and_duts['src_dut_index']
+        dst_dut_index = get_src_dst_asic_and_duts['dst_dut_index']
+        src_asic_index = get_src_dst_asic_and_duts['src_asic_index']
+        dst_asic_index = get_src_dst_asic_and_duts['dst_asic_index']
+        src_testPortIds = dutConfig["testPortIds"][src_dut_index][src_asic_index]
+        dst_testPortIds = dutConfig["testPortIds"][dst_dut_index][dst_asic_index]
+        testPortIds = src_testPortIds + list(set(dst_testPortIds) - set(src_testPortIds))
+
+        portIdNames = []
+        portIds = []
+
+        for idName in dutConfig["testPorts"]:
+            if re.match(r'(?:src|dst)_port\S+id', idName):
+                portIdNames.append(idName)
+                ipName = idName.replace('id', 'ip')
+                pytest_assert(
+                    ipName in dutConfig["testPorts"], 'Not find {} for {} in dutConfig'.format(ipName, idName))
+                portIds.append(dutConfig["testPorts"][idName])
+        pytest_assert(self.replaceNonExistentPortId(testPortIds, list(portIds)), "No enough test ports")
+        for idx, idName in enumerate(portIdNames):
+            dutConfig["testPorts"][idName] = portIds[idx]
+            ipName = idName.replace('id', 'ip')
+            if 'src' in ipName:
+                testPortIps = dutConfig["testPortIps"][src_dut_index][src_asic_index]
+            else:
+                testPortIps = dutConfig["testPortIps"][dst_dut_index][dst_asic_index]
+            dutConfig["testPorts"][ipName] = testPortIps[portIds[idx]]['peer_addr']
+
+        if qosParams is not None:
+            portIdNames = []
+            portNumbers = []
+            portIds = []
+            for idName in qosParams.keys():
+                if re.match(r'(?:src|dst)_port\S+ids?', idName):
+                    portIdNames.append(idName)
+                    ids = qosParams[idName]
+                    if isinstance(ids, list):
+                        portIds += ids
+                        # if it's port list, record number of pots
+                        portNumbers.append(len(ids))
+                    else:
+                        portIds.append(ids)
+                        # record None to indicate it's just one port
+                        portNumbers.append(None)
+            pytest_assert(self.replaceNonExistentPortId(testPortIds, portIds), "No enough test ports")
+            startPos = 0
+            for idx, idName in enumerate(portIdNames):
+                if portNumbers[idx] is not None:    # port list
+                    qosParams[idName] = [
+                        portId for portId in portIds[startPos:startPos + portNumbers[idx]]]
+                    startPos += portNumbers[idx]
+                else:   # not list, just one port
+                    qosParams[idName] = portIds[startPos]
+                    startPos += 1
+        logger.debug('updateTestPortIdIp dutConfig["testPorts"]: {}'.format(dutConfig["testPorts"]))
+
     @pytest.fixture(scope='module')
     def swapSyncd_on_selected_duts(self, request, duthosts, creds, tbinfo, lower_tor_host,  # noqa: F811
                                    core_dump_and_config_check):  # noqa: F811
@@ -1019,7 +1112,6 @@ def dutConfig(
         downlinkPortIps = []
         downlinkPortNames = []
         sysPortMap = {}
-
         src_dut_index = get_src_dst_asic_and_duts['src_dut_index']
         src_asic_index = get_src_dst_asic_and_duts['src_asic_index']
         src_dut = get_src_dst_asic_and_duts['src_dut']
@@ -1886,9 +1978,14 @@ def dutQosConfig(
                 portSpeedCableLength = dutQosConfig["portSpeedCableLength"]
         else:
             qosParams = qosConfigs['qos_params'][dutAsic][dutTopo]
+
+        # Get buffer config for all devices (not just cisco/broadcom)
+        bufferConfig = dutBufferConfig(duthost, dut_asic)
+
         yield {
             "param": qosParams,
             "portSpeedCableLength": portSpeedCableLength,
+            "bufferConfig": bufferConfig,
         }
 
     @pytest.fixture(scope='class')