Add background traffic for test_pfc_pause_extra_lossless test

Add background traffic for test_pfc_pause_extra_lossless test in
script tests/qos/test_tunnel_qos_remap.py

Author: echuawu

tests/conftest.py

@@ -40,6 +40,7 @@
 from tests.common.helpers.dut_ports import encode_dut_port_name
 from tests.common.helpers.dut_utils import encode_dut_and_container_name
 from tests.common.helpers.parallel_utils import InitialCheckState, InitialCheckStatus
+from tests.common.helpers.pfcwd_helper import TrafficPorts, select_test_ports, set_pfc_timers
 from tests.common.system_utils import docker
 from tests.common.testbed import TestbedInfo
 from tests.common.utilities import get_inventory_files, wait_until
@@ -2780,3 +2781,117 @@ def start_platform_api_service(duthosts, enum_rand_one_per_hwsku_hostname, local
 
         res = localhost.wait_for(host=dut_ip, port=SERVER_PORT, state='started', delay=1, timeout=10)
         assert res['failed'] is False
+
+
+def update_t1_test_ports(duthost, mg_facts, test_ports, tbinfo):
+    """
+    Find out active IP interfaces and use the list to
+    remove inactive ports from test_ports
+    """
+    ip_ifaces = duthost.get_active_ip_interfaces(tbinfo, asic_index=0)
+    port_list = []
+    for iface in list(ip_ifaces.keys()):
+        if iface.startswith("PortChannel"):
+            port_list.extend(
+                mg_facts["minigraph_portchannels"][iface]["members"]
+            )
+        else:
+            port_list.append(iface)
+    port_list_set = set(port_list)
+    for port in list(test_ports.keys()):
+        if port not in port_list_set:
+            del test_ports[port]
+    return test_ports
+
+
+@pytest.fixture(scope="module")
+def setup_pfc_test(
+    duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, conn_graph_facts, tbinfo,     # noqa F811
+):
+    """
+    Sets up all the parameters needed for the PFC Watchdog tests
+
+    Args:
+        duthost: AnsibleHost instance for DUT
+        ptfhost: AnsibleHost instance for PTF
+        conn_graph_facts: fixture that contains the parsed topology info
+
+    Yields:
+        setup_info: dictionary containing pfc timers, generated test ports and selected test ports
+    """
+    SUPPORTED_T1_TOPOS = {"t1-lag", "t1-64-lag", "t1-56-lag", "t1-28-lag", "t1-32-lag"}
+    duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
+    mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
+    port_list = list(mg_facts['minigraph_ports'].keys())
+    neighbors = conn_graph_facts['device_conn'].get(duthost.hostname, {})
+    dut_eth0_ip = duthost.mgmt_ip
+    vlan_nw = None
+
+    if mg_facts['minigraph_vlans']:
+        # Filter VLANs with one interface inside only(PortChannel interface in case of t0-56-po2vlan topo)
+        unexpected_vlans = []
+        for vlan, vlan_data in list(mg_facts['minigraph_vlans'].items()):
+            if len(vlan_data['members']) < 2:
+                unexpected_vlans.append(vlan)
+
+        # Update minigraph_vlan_interfaces with only expected VLAN interfaces
+        expected_vlan_ifaces = []
+        for vlan in unexpected_vlans:
+            for mg_vl_iface in mg_facts['minigraph_vlan_interfaces']:
+                if vlan != mg_vl_iface['attachto']:
+                    expected_vlan_ifaces.append(mg_vl_iface)
+        if expected_vlan_ifaces:
+            mg_facts['minigraph_vlan_interfaces'] = expected_vlan_ifaces
+
+        # gather all vlan specific info
+        vlan_addr = mg_facts['minigraph_vlan_interfaces'][0]['addr']
+        vlan_prefix = mg_facts['minigraph_vlan_interfaces'][0]['prefixlen']
+        vlan_dev = mg_facts['minigraph_vlan_interfaces'][0]['attachto']
+        vlan_ips = duthost.get_ip_in_range(
+            num=1, prefix="{}/{}".format(vlan_addr, vlan_prefix),
+            exclude_ips=[vlan_addr])['ansible_facts']['generated_ips']
+        vlan_nw = vlan_ips[0].split('/')[0]
+
+    # build the port list for the test
+    tp_handle = TrafficPorts(mg_facts, neighbors, vlan_nw)
+    test_ports = tp_handle.build_port_list()
+
+    # In T1 topology update test ports by removing inactive ports
+    topo = tbinfo["topo"]["name"]
+    if topo in SUPPORTED_T1_TOPOS:
+        test_ports = update_t1_test_ports(
+            duthost, mg_facts, test_ports, tbinfo
+        )
+    # select a subset of ports from the generated port list
+    selected_ports = select_test_ports(test_ports)
+
+    setup_info = {'test_ports': test_ports,
+                  'port_list': port_list,
+                  'selected_test_ports': selected_ports,
+                  'pfc_timers': set_pfc_timers(),
+                  'neighbors': neighbors,
+                  'eth0_ip': dut_eth0_ip
+                  }
+
+    if mg_facts['minigraph_vlans']:
+        setup_info['vlan'] = {'addr': vlan_addr,
+                              'prefix': vlan_prefix,
+                              'dev': vlan_dev
+                              }
+    else:
+        setup_info['vlan'] = None
+
+    # stop pfcwd
+    logger.info("--- Stopping Pfcwd ---")
+    duthost.command("pfcwd stop")
+
+    # set poll interval
+    duthost.command("pfcwd interval {}".format(setup_info['pfc_timers']['pfc_wd_poll_time']))
+
+    # set bulk counter chunk size
+    logger.info("--- Setting bulk counter polling chunk size ---")
+    duthost.command('redis-cli -n 4 hset "FLEX_COUNTER_TABLE|PORT" BULK_CHUNK_SIZE 64'
+                    ' BULK_CHUNK_SIZE_PER_PREFIX "SAI_PORT_STAT_IF_OUT_QLEN:0;SAI_PORT_STAT_IF_IN_FEC:32"')
+
+    logger.info("setup_info : {}".format(setup_info))
+    yield setup_info

tests/pfcwd/conftest.py

@@ -10,7 +10,6 @@
 from tests.common.fixtures.ptfhost_utils import pause_garp_service          # noqa F401
 from tests.common.mellanox_data import is_mellanox_device as isMellanoxDevice
 from tests.common.cisco_data import is_cisco_device
-from tests.common.helpers.pfcwd_helper import TrafficPorts, set_pfc_timers, select_test_ports
 from tests.common.utilities import str2bool
 
 logger = logging.getLogger(__name__)
@@ -80,117 +79,6 @@ def fake_storm(request, duthosts, enum_rand_one_per_hwsku_frontend_hostname):
         else request.config.getoption('--fake-storm')
 
 
-def update_t1_test_ports(duthost, mg_facts, test_ports, tbinfo):
-    """
-    Find out active IP interfaces and use the list to
-    remove inactive ports from test_ports
-    """
-    ip_ifaces = duthost.get_active_ip_interfaces(tbinfo, asic_index=0)
-    port_list = []
-    for iface in list(ip_ifaces.keys()):
-        if iface.startswith("PortChannel"):
-            port_list.extend(
-                mg_facts["minigraph_portchannels"][iface]["members"]
-            )
-        else:
-            port_list.append(iface)
-    port_list_set = set(port_list)
-    for port in list(test_ports.keys()):
-        if port not in port_list_set:
-            del test_ports[port]
-    return test_ports
-
-
-@pytest.fixture(scope="module")
-def setup_pfc_test(
-    duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, conn_graph_facts, tbinfo,     # noqa F811
-):
-    """
-    Sets up all the parameters needed for the PFC Watchdog tests
-
-    Args:
-        duthost: AnsibleHost instance for DUT
-        ptfhost: AnsibleHost instance for PTF
-        conn_graph_facts: fixture that contains the parsed topology info
-
-    Yields:
-        setup_info: dictionary containing pfc timers, generated test ports and selected test ports
-    """
-    SUPPORTED_T1_TOPOS = {"t1-lag", "t1-64-lag", "t1-56-lag", "t1-28-lag", "t1-32-lag"}
-    duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
-    mg_facts = duthost.get_extended_minigraph_facts(tbinfo)
-    port_list = list(mg_facts['minigraph_ports'].keys())
-    neighbors = conn_graph_facts['device_conn'].get(duthost.hostname, {})
-    dut_eth0_ip = duthost.mgmt_ip
-    vlan_nw = None
-
-    if mg_facts['minigraph_vlans']:
-        # Filter VLANs with one interface inside only(PortChannel interface in case of t0-56-po2vlan topo)
-        unexpected_vlans = []
-        for vlan, vlan_data in list(mg_facts['minigraph_vlans'].items()):
-            if len(vlan_data['members']) < 2:
-                unexpected_vlans.append(vlan)
-
-        # Update minigraph_vlan_interfaces with only expected VLAN interfaces
-        expected_vlan_ifaces = []
-        for vlan in unexpected_vlans:
-            for mg_vl_iface in mg_facts['minigraph_vlan_interfaces']:
-                if vlan != mg_vl_iface['attachto']:
-                    expected_vlan_ifaces.append(mg_vl_iface)
-        if expected_vlan_ifaces:
-            mg_facts['minigraph_vlan_interfaces'] = expected_vlan_ifaces
-
-        # gather all vlan specific info
-        vlan_addr = mg_facts['minigraph_vlan_interfaces'][0]['addr']
-        vlan_prefix = mg_facts['minigraph_vlan_interfaces'][0]['prefixlen']
-        vlan_dev = mg_facts['minigraph_vlan_interfaces'][0]['attachto']
-        vlan_ips = duthost.get_ip_in_range(
-            num=1, prefix="{}/{}".format(vlan_addr, vlan_prefix),
-            exclude_ips=[vlan_addr])['ansible_facts']['generated_ips']
-        vlan_nw = vlan_ips[0].split('/')[0]
-
-    topo = tbinfo["topo"]["name"]
-
-    # build the port list for the test
-    config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts']
-    tp_handle = TrafficPorts(mg_facts, neighbors, vlan_nw, topo, config_facts)
-    test_ports = tp_handle.build_port_list()
-
-    # In T1 topology update test ports by removing inactive ports
-    if topo in SUPPORTED_T1_TOPOS:
-        test_ports = update_t1_test_ports(
-            duthost, mg_facts, test_ports, tbinfo
-        )
-    # select a subset of ports from the generated port list
-    selected_ports = select_test_ports(test_ports)
-
-    setup_info = {'test_ports': test_ports,
-                  'port_list': port_list,
-                  'selected_test_ports': selected_ports,
-                  'pfc_timers': set_pfc_timers(),
-                  'neighbors': neighbors,
-                  'eth0_ip': dut_eth0_ip
-                  }
-
-    if mg_facts['minigraph_vlans']:
-        setup_info['vlan'] = {'addr': vlan_addr,
-                              'prefix': vlan_prefix,
-                              'dev': vlan_dev
-                              }
-    else:
-        setup_info['vlan'] = None
-
-    # stop pfcwd
-    logger.info("--- Stopping Pfcwd ---")
-    duthost.command("pfcwd stop")
-
-    # set poll interval
-    duthost.command("pfcwd interval {}".format(setup_info['pfc_timers']['pfc_wd_poll_time']))
-
-    logger.info("setup_info : {}".format(setup_info))
-    yield setup_info
-
-
 @pytest.fixture(scope="module")
 def setup_dut_test_params(
     duthosts, enum_rand_one_per_hwsku_frontend_hostname, ptfhost, conn_graph_facts, tbinfo,     # noqa F811

tests/qos/test_tunnel_qos_remap.py

@@ -34,6 +34,7 @@
 from ptf.testutils import simple_tcp_packet
 from tests.common.fixtures.conn_graph_facts import conn_graph_facts, fanout_graph_facts     # noqa F401
 from tests.common.helpers.pfc_storm import PFCStorm
+from tests.common.helpers.pfcwd_helper import send_background_traffic
 
 
 pytestmark = [
@@ -354,32 +355,27 @@ def test_separated_qos_map_on_tor(ptfhost, rand_selected_dut, rand_unselected_du
         counter_poll_config(rand_selected_dut, 'queue', 10000)
 
 
-def pfc_pause_test(storm_handler, peer_info, prio, ptfadapter, dut, port, queue, pkt, src_port, exp_pkt, dst_ports):
+def pfc_pause_test(ptfhost, storm_handler, peer_info, prio, ptfadapter, dut, port, queue, pkt, src_port, exp_pkt,
+                   dst_ports, test_ports_info):
     try:
-        # Start PFC storm from leaf fanout switch
-        start_pfc_storm(storm_handler, peer_info, prio)
-        ptfadapter.dataplane.flush()
-        # Record the queue counter before sending test packet
-        base_queue_count = get_queue_counter(dut, port, queue, False)   # noqa F841
-        # Send testing packet again
-        testutils.send_packet(ptfadapter, src_port, pkt, 1)
-        # The packet should be paused
-        testutils.verify_no_packet_any(ptfadapter, exp_pkt, dst_ports)
-        # Check the queue counter didn't increase
-        queue_count = get_queue_counter(dut, port, queue, False)        # noqa F841
-        # after 10 sec delay in queue counter reading, pfc frames sending might actually had already stopped.
-        # so bounce back packet might still send out, and queue counter increased accordingly.
-        # and then caused flaky test faiure.
-        # temporarily disable the assert queue counter here until find a better solution,
-        # such as reading counter using sai thrift API
-        # assert base_queue_count == queue_count
-        return True
+            start_pfc_storm(storm_handler, peer_info, prio)
+            ptfadapter.dataplane.flush()
+            # Record the queue counter before sending test packet
+            base_queue_count = get_queue_counter(dut, port, queue, True)
+            # Send testing packet again
+            testutils.send_packet(ptfadapter, src_port, pkt, 1)
+            # The packet should be paused
+            testutils.verify_no_packet_any(ptfadapter, exp_pkt, dst_ports)
+            # Check the queue counter didn't increase
+            queue_count = get_queue_counter(dut, port, queue, False)
+            assert base_queue_count == queue_count
+            return True
     finally:
         stop_pfc_storm(storm_handler)
 
 
 def test_pfc_pause_extra_lossless_standby(ptfhost, fanouthosts, rand_selected_dut, rand_unselected_dut,
-                                          setup_standby_ports_on_rand_selected_tor,
+                                          setup_standby_ports_on_rand_selected_tor, setup_pfc_test,  # noqa F811
         toggle_all_simulator_ports_to_rand_unselected_tor, tbinfo, ptfadapter, conn_graph_facts, fanout_graph_facts, dut_config): # noqa F811
     """
     The test case is to verify PFC pause frame can pause extra lossless queues in dualtor deployment.
@@ -389,6 +385,7 @@ def test_pfc_pause_extra_lossless_standby(ptfhost, fanouthosts, rand_selected_du
     3. Generate PFC pause on fanout switch (T1 ports)
     4. Verify lossless traffic are paused
     """
+    setup_info = setup_pfc_test
     if "cisco-8000" in dut_config["asic_type"]:
         pytest.skip("Replacing test with test_pfc_watermark_extra_lossless_standby for Cisco-8000.")
     TEST_DATA = {
@@ -440,8 +437,8 @@ def test_pfc_pause_extra_lossless_standby(ptfhost, fanouthosts, rand_selected_du
         retry = 0
         while retry < PFC_PAUSE_TEST_RETRY_MAX:
             try:
-                if pfc_pause_test(storm_handler, peer_info, prio, ptfadapter, rand_selected_dut, actual_port_name,
-                                  queue, pkt, src_port, exp_pkt, dst_ports):
+                if pfc_pause_test(ptfhost, storm_handler, peer_info, prio, ptfadapter, rand_selected_dut,
+                                  actual_port_name, queue, pkt, src_port, exp_pkt, dst_ports, setup_info['test_ports']):
                     break
             except AssertionError as err:
                 retry += 1
@@ -457,7 +454,7 @@ def test_pfc_pause_extra_lossless_standby(ptfhost, fanouthosts, rand_selected_du
 
 
 def test_pfc_pause_extra_lossless_active(ptfhost, fanouthosts, rand_selected_dut, rand_unselected_dut,
-                                         setup_standby_ports_on_rand_unselected_tor,
+                                         setup_standby_ports_on_rand_unselected_tor, setup_pfc_test,  # noqa F811
         toggle_all_simulator_ports_to_rand_selected_tor, tbinfo, ptfadapter, conn_graph_facts, fanout_graph_facts, dut_config): # noqa F811
     """
     The test case is to verify PFC pause frame can pause extra lossless queues in dualtor deployment.
@@ -467,6 +464,7 @@ def test_pfc_pause_extra_lossless_active(ptfhost, fanouthosts, rand_selected_dut
     3. Generate PFC pause on fanout switch (Server facing ports)
     4. Verify lossless traffic are paused
     """
+    setup_info = setup_pfc_test
     if "cisco-8000" in dut_config["asic_type"]:
         pytest.skip("Replacing test with test_pfc_watermark_extra_lossless_active for Cisco-8000.")
     TEST_DATA = {
@@ -516,9 +514,9 @@ def test_pfc_pause_extra_lossless_active(ptfhost, fanouthosts, rand_selected_dut
         retry = 0
         while retry < PFC_PAUSE_TEST_RETRY_MAX:
             try:
-                if pfc_pause_test(storm_handler, peer_info, prio, ptfadapter, rand_selected_dut,
+                if pfc_pause_test(ptfhost, storm_handler, peer_info, prio, ptfadapter, rand_selected_dut,
                                   dualtor_meta['selected_port'], queue, tunnel_pkt.exp_pkt, src_port, exp_pkt,
-                                  dst_ports):
+                                  dst_ports, setup_info['test_ports']):
                     break
             except AssertionError as err:
                 retry += 1