Enhance PFC watchdog log analyzer to check debug information on a per-vendor basis (sonic-net#10305)

* Enhance PFC watchdog debug information checking

1. Check diagnosis information in PFC watchdog detection message on a per-vendor basis
2. Move expected log message to a common file

Author: stephenxs

tests/pfcwd/files/pfcwd_helper.py

@@ -8,6 +8,15 @@
 if sys.version_info[0] >= 3:
     unicode = str
 
+EXPECT_PFC_WD_DETECT_RE = ".* detected PFC storm .*"
+VENDOR_SPEC_ADDITIONAL_INFO_RE = {
+    "mellanox":
+        r"additional info: occupancy:[0-9]+\|packets:[0-9]+\|packets_last:[0-9]+\|pfc_rx_packets:[0-9]+\|"
+        r"pfc_rx_packets_last:[0-9]+\|pfc_duration:[0-9]+\|pfc_duration_last:[0-9]+\|timestamp:[0-9]+\.[0-9]+\|"
+        r"timestamp_last:[0-9]+\.[0-9]+\|real_poll_time:[0-9]+"
+    }
+EXPECT_PFC_WD_RESTORE_RE = ".*storm restored.*"
+
 
 class TrafficPorts(object):
     """ Generate a list of ports needed for the PFC Watchdog test"""
@@ -342,3 +351,16 @@ def start_wd_on_ports(duthost, port, restore_time, detect_time, action="drop"):
     """
     duthost.command("pfcwd start --action {} --restoration-time {} {} {}"
                     .format(action, restore_time, port, detect_time))
+
+
+def fetch_vendor_specific_diagnosis_re(duthost):
+    """
+    Fetch regular expression of vendor specific diagnosis information
+    Args:
+        duthost: The duthost object
+    """
+    unsupported_branches = ['202012', '202205', '202211']
+    if duthost.os_version in unsupported_branches or duthost.sonic_release in unsupported_branches:
+        return ""
+
+    return VENDOR_SPEC_ADDITIONAL_INFO_RE.get(duthost.facts["asic_type"], "")

tests/pfcwd/test_pfcwd_all_port_storm.py

@@ -7,10 +7,9 @@
 from tests.common.helpers.pfc_storm import PFCMultiStorm
 from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer
 from .files.pfcwd_helper import start_wd_on_ports
+from .files.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE, fetch_vendor_specific_diagnosis_re
 
 TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates")
-EXPECT_PFC_WD_DETECT_RE = ".* detected PFC storm .*"
-EXPECT_PFC_WD_RESTORE_RE = ".*storm restored.*"
 
 pytestmark = [
     pytest.mark.disable_loganalyzer,
@@ -157,7 +156,10 @@ def test_all_port_storm_restore(self, duthosts, enum_rand_one_per_hwsku_frontend
         duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
         storm_hndle = storm_test_setup_restore
         logger.info("--- Testing if PFC storm is detected on all ports ---")
-        self.run_test(duthost, storm_hndle, expect_regex=[EXPECT_PFC_WD_DETECT_RE], syslog_marker="all_port_storm",
+        self.run_test(duthost,
+                      storm_hndle,
+                      expect_regex=[EXPECT_PFC_WD_DETECT_RE + fetch_vendor_specific_diagnosis_re(duthost)],
+                      syslog_marker="all_port_storm",
                       action="storm")
 
         logger.info("--- Testing if PFC storm is restored on all ports ---")

tests/pfcwd/test_pfcwd_function.py

@@ -10,6 +10,7 @@
 from tests.common.helpers.pfc_storm import PFCStorm
 from tests.common.plugins.loganalyzer.loganalyzer import LogAnalyzer
 from .files.pfcwd_helper import start_wd_on_ports
+from .files.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE, fetch_vendor_specific_diagnosis_re
 from tests.ptf_runner import ptf_runner
 from tests.common import port_toggle
 from tests.common import constants
@@ -20,8 +21,6 @@
 PTF_PORT_MAPPING_MODE = 'use_orig_interface'
 
 TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates")
-EXPECT_PFC_WD_DETECT_RE = ".* detected PFC storm .*"
-EXPECT_PFC_WD_RESTORE_RE = ".*storm restored.*"
 WD_ACTION_MSG_PFX = {"dontcare": "Verify PFCWD detection when queue buffer is not empty "
                      "and proper function of pfcwd drop action",
                      "drop": "Verify proper function of pfcwd drop action",
@@ -671,7 +670,7 @@ def storm_detect_path(self, dut, port, action):
         reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
         loganalyzer.ignore_regex.extend(reg_exp)
         loganalyzer.expect_regex = []
-        loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE])
+        loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE + fetch_vendor_specific_diagnosis_re(dut)])
         loganalyzer.match_regex = []
 
         if action != "dontcare":
@@ -1067,7 +1066,7 @@ def test_pfcwd_port_toggle(self, request, fake_storm, setup_pfc_test, setup_dut_
                 reg_exp = loganalyzer.parse_regexp_file(src=ignore_file)
                 loganalyzer.ignore_regex.extend(reg_exp)
                 loganalyzer.expect_regex = []
-                loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE])
+                loganalyzer.expect_regex.extend([EXPECT_PFC_WD_DETECT_RE + fetch_vendor_specific_diagnosis_re(duthost)])
                 loganalyzer.match_regex = []
 
                 port_toggle(self.dut, tbinfo, ports=[port])

tests/pfcwd/test_pfcwd_warm_reboot.py

@@ -16,10 +16,9 @@
 from tests.common.utilities import InterruptableThread
 from tests.common.utilities import join_all
 from tests.ptf_runner import ptf_runner
+from .files.pfcwd_helper import EXPECT_PFC_WD_DETECT_RE, EXPECT_PFC_WD_RESTORE_RE
 
 TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates")
-EXPECT_PFC_WD_DETECT_RE = ".* detected PFC storm .*"
-EXPECT_PFC_WD_RESTORE_RE = ".*storm restored.*"
 TESTCASE_INFO = {'no_storm': {'test_sequence': ["detect", "restore", "warm-reboot", "detect", "restore"],
                               'desc': "Test PFC storm detect/restore before and after warm boot"},
                  'storm': {'test_sequence': ["detect", "warm-reboot", "detect", "restore"],