diff --git a/tests/common/helpers/pfc_gen.py b/tests/common/helpers/pfc_gen.py index dfbda38d338..de87f7305c7 100755 --- a/tests/common/helpers/pfc_gen.py +++ b/tests/common/helpers/pfc_gen.py @@ -17,15 +17,9 @@ logger = logging.getLogger('MyLogger') logger.setLevel(logging.DEBUG) -# Minimum number of processes to be created -MIN_PROCESS_NUM = 2 - # Maximum number of processes to be created MAX_PROCESS_NUM = 4 -# Minimum number of packets for enabling multiple processes -MIN_PACKET_NUM_MP = 10000 - class PacketSender(): """ @@ -180,13 +174,6 @@ def main(): pre_str = 'GLOBAL_PF' if options.global_pf else 'PFC' logger.debug(pre_str + '_STORM_START') - # Send PFC pause with multiple processes even if only one interface is provided - # if packet number is smaller than the threshold, then it's not necessary to use multiple processes - if options.num >= MIN_PACKET_NUM_MP: - while len(interfaces) < MIN_PROCESS_NUM: - interfaces.extend(interfaces) - options.num /= 2 - # Start sending PFC pause frames senders = [] interface_slices = [[] for i in range(MAX_PROCESS_NUM)] diff --git a/tests/pfcwd/files/pfcwd_helper.py b/tests/pfcwd/files/pfcwd_helper.py index 0b5a8b12a7f..898b0a1d378 100644 --- a/tests/pfcwd/files/pfcwd_helper.py +++ b/tests/pfcwd/files/pfcwd_helper.py @@ -5,6 +5,7 @@ import pytest import contextlib import time +import logging from tests.ptf_runner import ptf_runner from tests.common import constants @@ -24,6 +25,8 @@ EXPECT_PFC_WD_RESTORE_RE = ".*storm restored.*" +logger = logging.getLogger(__name__) + class TrafficPorts(object): """ Generate a list of ports needed for the PFC Watchdog test""" @@ -541,3 +544,17 @@ def _stop_background_traffic(ptfhost, background_traffic_log): pids = ptfhost.shell(f"pgrep -f {background_traffic_log}")["stdout_lines"] for pid in pids: ptfhost.shell(f"kill -9 {pid}", module_ignore_errors=True) + + +def check_pfc_storm_state(dut, port, queue): + """ + Helper function to check if PFC storm is detected/restored on a given queue + """ + pfcwd_stats = dut.show_and_parse("show pfcwd stats") + queue_name = str(port) + ":" + str(queue) + for entry in pfcwd_stats: + if entry["queue"] == queue_name: + logger.info("PFCWD status on queue {} stats: {}".format(queue_name, entry)) + return entry['storm detected/restored'] + logger.info("PFCWD not triggered on queue {}".format(queue_name)) + return None diff --git a/tests/pfcwd/test_pfcwd_function.py b/tests/pfcwd/test_pfcwd_function.py index 987db273745..4e9e6d29ba3 100644 --- a/tests/pfcwd/test_pfcwd_function.py +++ b/tests/pfcwd/test_pfcwd_function.py @@ -17,7 +17,8 @@ from tests.common import constants from tests.common.dualtor.dual_tor_utils import is_tunnel_qos_remap_enabled, dualtor_ports # noqa F401 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_m # noqa F401, E501 -from .files.pfcwd_helper import send_background_traffic +from .files.pfcwd_helper import send_background_traffic, check_pfc_storm_state +from tests.common.utilities import wait_until PTF_PORT_MAPPING_MODE = 'use_orig_interface' @@ -712,6 +713,10 @@ def storm_detect_path(self, dut, port, action): test_ports_info = {self.pfc_wd['rx_port'][0]: self.pfc_wd} queues = [self.storm_hndle.pfc_queue_idx] + if dut.facts['asic_type'] == "mellanox": + PFC_STORM_TIMEOUT = 30 + pfcwd_stats_before_test = check_pfc_storm_state(dut, port, self.storm_hndle.pfc_queue_idx) + with send_background_traffic(dut, self.ptf, queues, selected_test_ports, test_ports_info): if action != "dontcare": start_wd_on_ports(dut, port, restore_time, detect_time, action) @@ -728,7 +733,14 @@ def storm_detect_path(self, dut, port, action): if self.pfc_wd['fake_storm']: PfcCmd.set_storm_status(dut, self.queue_oid, "enabled") - time.sleep(5) + if dut.facts['asic_type'] == "mellanox": + # On Mellanox platform, more time is required for PFC storm being triggered + # as PFC pause sent from Non-Mellanox leaf fanout is not continuous sometimes. + pytest_assert(wait_until(PFC_STORM_TIMEOUT, 2, 0, + lambda: check_pfc_storm_state(dut, port, self.storm_hndle.pfc_queue_idx) != pfcwd_stats_before_test), # noqa: E501, E128 + "PFC storm state did not change as expected") # noqa: E127 + else: + time.sleep(5) # storm detect logger.info("Verify if PFC storm is detected on port {}".format(port))