From 091931710a4a56489f75dc5fec006813d5d8cb23 Mon Sep 17 00:00:00 2001 From: "Austin (Thang Pham)" Date: Wed, 29 Jan 2025 16:49:45 +1100 Subject: [PATCH] chore: update multidut pfcwd tests (#16657) Summary: Update the missing T0 T1 tests feature in multidut Fixes # (issue) 30929279 The following PR were migrated for pfcwd [Snappi] Increase number of calls for ARP wait #7370 [Snappi] PFCWD Tolerance Fixes #9567 [Snappi] Modifications to pfcwd a2a test design for mlnx 4600c #7367 to multidut code [Snappi] Modified pfcwd tests to add warm-up traffic #7712 Signed-off-by: Austin Pham --- .../pfcwd_multidut_burst_storm_helper.py | 4 +- .../files/pfcwd_multidut_multi_node_helper.py | 45 ++++--- .../pfcwd_multidut_runtime_traffic_helper.py | 113 ++++++++++-------- .../test_multidut_pfcwd_a2a_with_snappi.py | 2 +- .../test_multidut_pfcwd_m2o_with_snappi.py | 2 +- tests/snappi_tests/test_multidut_snappi.py | 2 +- 6 files changed, 100 insertions(+), 68 deletions(-) diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py index afa8feff005..55af85a03e2 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py @@ -247,7 +247,7 @@ def __gen_traffic(testbed_config, pause_pkt.pause_class_6.value = pause_time[6] pause_pkt.pause_class_7.value = pause_time[7] - pause_flow_start_time = id * (pause_flow_dur_sec + pause_flow_gap_sec) + pause_flow_start_time = id * (pause_flow_dur_sec + pause_flow_gap_sec) + WARM_UP_TRAFFIC_DUR pause_flow.rate.pps = pause_pps pause_flow.size.fixed = 64 @@ -275,7 +275,7 @@ def __run_traffic(api, config, all_flow_names, exp_dur_sec): api.set_config(config) logger.info('Wait for Arp to Resolve ...') - wait_for_arp(api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(api, max_attempts=30, poll_interval_sec=2) logger.info('Starting transmit on all flows ...') ts = api.transmit_state() diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py index 4a20224615f..742608c82a5 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py @@ -153,6 +153,11 @@ def run_pfcwd_multi_node_test(api, all_flow_names=all_flow_names, exp_dur_sec=exp_dur_sec) + """ Retrieve ASIC information for DUT """ + asic_type = egress_duthost.facts['asic_type'] + + rx_tx_tol_thrhlds = [0.0001, 0.0002] # Maintain a 0.01% and 0.02% deviation between tx and rx frames + __verify_results(rows=flow_stats, speed_gbps=speed_gbps, pause_flow_name=PAUSE_FLOW_NAME, @@ -164,7 +169,9 @@ def run_pfcwd_multi_node_test(api, data_pkt_size=DATA_PKT_SIZE, trigger_pfcwd=trigger_pfcwd, pause_port_id=rx_port_id_list[0], - tolerance=TOLERANCE_THRESHOLD) + rx_deviation=TOLERANCE_THRESHOLD, + rx_tx_deviations=rx_tx_tol_thrhlds, + asic_type=asic_type) def __data_flow_name(name_prefix, src_id, dst_id, prio): @@ -532,7 +539,7 @@ def __run_traffic(api, config, all_flow_names, exp_dur_sec): api.set_config(config) logger.info('Wait for Arp to Resolve ...') - wait_for_arp(api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(api, max_attempts=30, poll_interval_sec=2) logger.info('Starting transmit on all flows ...') ts = api.transmit_state() @@ -586,7 +593,9 @@ def __verify_results(rows, data_pkt_size, trigger_pfcwd, pause_port_id, - tolerance): + rx_deviation, + rx_tx_deviations, + asic_type): """ Verify if we get expected experiment results @@ -602,11 +611,16 @@ def __verify_results(rows, test_flow_pause (bool): if test flows are expected to be paused trigger_pfcwd (bool): if PFC watchdog is expected to be triggered pause_port_id (int): ID of the port to send PFC pause frames - tolerance (float): maximum allowable deviation + rx_deviation (float): maximum allowable deviation for rx_frames relative to theoretical value + rx_tx_deviations (list of floats): maximum allowable % deviation for rx_frames relative to tx_frames Returns: N/A """ + + """ Check for whether DUT is a Mellanox device """ + is_mlnx_device = True if "mellanox" in asic_type.lower() else False + for row in rows: flow_name = row.name tx_frames = row.frames_tx @@ -629,7 +643,7 @@ def __verify_results(rows, exp_bg_flow_rx_pkts = bg_flow_rate_percent / 100.0 * speed_gbps \ * 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size deviation = (rx_frames - exp_bg_flow_rx_pkts) / float(exp_bg_flow_rx_pkts) - pytest_assert(abs(deviation) < tolerance, + pytest_assert(abs(deviation) < rx_deviation, '{} should receive {} packets (actual {})'. format(flow_name, exp_bg_flow_rx_pkts, rx_frames)) @@ -641,14 +655,17 @@ def __verify_results(rows, exp_test_flow_rx_pkts = test_flow_rate_percent / 100.0 * speed_gbps \ * 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size - if trigger_pfcwd and\ - (src_port_id == pause_port_id or dst_port_id == pause_port_id): + if trigger_pfcwd and dst_port_id == pause_port_id: """ Once PFC watchdog is triggered, it will impact bi-directional traffic """ logger.info('Once PFC watchdog is triggered, it will impact bi-directional traffic') logger.info('Tx and Rx should have dropped packets') pytest_assert(tx_frames > rx_frames, '{} should have dropped packets'.format(flow_name)) - + elif trigger_pfcwd and src_port_id == pause_port_id: + if is_mlnx_device: + """ During a pfc storm with pfcwd triggered, Mellanox devices do not drop Rx packets """ + pytest_assert(tx_frames == rx_frames, + '{} should not have dropped packets for Mellanox device'.format(flow_name)) elif not trigger_pfcwd and dst_port_id == pause_port_id: """ This test flow is delayed by PFC storm """ logger.info('This test flow is delayed by PFC storm') @@ -660,14 +677,12 @@ def __verify_results(rows, format(flow_name, exp_test_flow_rx_pkts, rx_frames)) else: - """ Otherwise, the test flow is not impacted by PFC storm """ - logger.info('the test flow is not impacted by PFC storm') - logger.info('Tx and Rx should not have any dropped packet') - - pytest_assert(tx_frames == rx_frames, - '{} should not have any dropped packet'.format(flow_name)) + for dev_pct in rx_tx_deviations: + """ Otherwise, the test flow is not impacted by PFC storm """ + pytest_assert(abs(tx_frames - rx_frames)/float(tx_frames) < dev_pct, + '{} should be within {} percent deviation'.format(flow_name, dev_pct*100)) deviation = (rx_frames - exp_test_flow_rx_pkts) / float(exp_test_flow_rx_pkts) - pytest_assert(abs(deviation) < tolerance, + pytest_assert(abs(deviation) < rx_deviation, '{} should receive {} packets (actual {})'. format(flow_name, exp_test_flow_rx_pkts, rx_frames)) diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py index 385536db07f..a1ad07e3f42 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py @@ -3,16 +3,18 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 -from tests.common.snappi_tests.common_helpers import start_pfcwd, stop_pfcwd +from tests.common.snappi_tests.common_helpers import start_pfcwd, stop_pfcwd, sec_to_nanosec from tests.common.snappi_tests.port import select_ports, select_tx_port # noqa: F401 from tests.common.snappi_tests.snappi_helpers import wait_for_arp from tests.common.snappi_tests.snappi_test_params import SnappiTestParams from tests.common.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict DATA_FLOW_NAME = "Data Flow" +WARM_UP_TRAFFIC_NAME = "Warm Up Traffic" DATA_PKT_SIZE = 1024 DATA_FLOW_DURATION_SEC = 15 -PFCWD_START_DELAY_SEC = 3 +WARM_UP_TRAFFIC_DUR = 1 +PFCWD_START_DELAY_SEC = 3 + WARM_UP_TRAFFIC_DUR SNAPPI_POLL_DELAY_SEC = 2 TOLERANCE_THRESHOLD = 0.05 UDP_PORT_START = 5000 @@ -65,12 +67,20 @@ def run_pfcwd_runtime_traffic_test(api, stop_pfcwd(egress_duthost, rx_port['asic_value']) stop_pfcwd(ingress_duthost, tx_port['asic_value']) + """ Warm up traffic is initially sent before any other traffic to prevent pfcwd + fake alerts caused by idle links (non-incremented packet counters) during pfcwd detection periods """ + warm_up_traffic_dur_sec = WARM_UP_TRAFFIC_DUR + warm_up_traffic_delay_sec = 0 + __gen_traffic(testbed_config=testbed_config, port_config_list=port_config_list, tx_port_id=tx_port_id, rx_port_id=rx_port_id, - data_flow_name=DATA_FLOW_NAME, - data_flow_dur_sec=DATA_FLOW_DURATION_SEC, + data_flow_name_list=[WARM_UP_TRAFFIC_NAME, DATA_FLOW_NAME], + data_flow_delay_sec_list=[ + warm_up_traffic_delay_sec, WARM_UP_TRAFFIC_DUR], + data_flow_dur_sec_list=[ + warm_up_traffic_dur_sec, DATA_FLOW_DURATION_SEC], data_pkt_size=DATA_PKT_SIZE, prio_list=prio_list, prio_dscp_map=prio_dscp_map) @@ -90,7 +100,8 @@ def run_pfcwd_runtime_traffic_test(api, speed_str = testbed_config.layer1[0].speed speed_gbps = int(speed_str.split('_')[1]) - __verify_results(rows=flow_stats, + data_flows = [flow_stat for flow_stat in flow_stats if DATA_FLOW_NAME in flow_stat.name] + __verify_results(rows=data_flows, speed_gbps=speed_gbps, data_flow_dur_sec=DATA_FLOW_DURATION_SEC, data_pkt_size=DATA_PKT_SIZE, @@ -101,8 +112,9 @@ def __gen_traffic(testbed_config, port_config_list, tx_port_id, rx_port_id, - data_flow_name, - data_flow_dur_sec, + data_flow_name_list, + data_flow_delay_sec_list, + data_flow_dur_sec_list, data_pkt_size, prio_list, prio_dscp_map): @@ -113,8 +125,9 @@ def __gen_traffic(testbed_config, testbed_config (obj): testbed L1/L2/L3 configuration port_config_list (list): list of port configuration port_id (int): ID of DUT port to test. - data_flow_name (str): data flow name - data_flow_dur_sec (int): duration of data flows in second + data_flow_name_list (list): list of data flow names + data_flow_delay_sec_list (list): list of data flow start delays in second + data_flow_dur_sec_list (list): list of data flow durations in second data_pkt_size (int): size of data packets in byte prio_list (list): priorities of data flows prio_dscp_map (dict): Priority vs. DSCP map (key = priority). @@ -136,44 +149,48 @@ def __gen_traffic(testbed_config, tx_port_name = testbed_config.ports[tx_port_id].name rx_port_name = testbed_config.ports[rx_port_id].name data_flow_rate_percent = int(100 / len(prio_list)) - - """ For each priority """ - for prio in prio_list: - data_flow = testbed_config.flows.flow( - name='{} Prio {}'.format(data_flow_name, prio))[-1] - - data_flow.tx_rx.port.tx_name = tx_port_name - data_flow.tx_rx.port.rx_name = rx_port_name - - eth, ipv4, udp = data_flow.packet.ethernet().ipv4().udp() - - eth.src.value = tx_mac - eth.dst.value = rx_mac - if pfcQueueGroupSize == 8: - eth.pfc_queue.value = prio - else: - eth.pfc_queue.value = pfcQueueValueDict[prio] - - global UDP_PORT_START - src_port = UDP_PORT_START - UDP_PORT_START += 1 - udp.src_port.increment.start = src_port - udp.src_port.increment.step = 1 - udp.src_port.increment.count = 1 - - ipv4.src.value = tx_port_config.ip - ipv4.dst.value = rx_port_config.ip - ipv4.priority.choice = ipv4.priority.DSCP - ipv4.priority.dscp.phb.values = prio_dscp_map[prio] - # ipv4.priority.dscp.ecn.value = ( - # ipv4.priority.dscp.ecn.CAPABLE_TRANSPORT_1) - - data_flow.size.fixed = data_pkt_size - data_flow.rate.percentage = data_flow_rate_percent - data_flow.duration.fixed_seconds.seconds = data_flow_dur_sec - - data_flow.metrics.enable = True - data_flow.metrics.loss = True + """ For each data flow """ + for i in range(len(data_flow_name_list)): + """ For each priority """ + for prio in prio_list: + data_flow = testbed_config.flows.flow( + name='{} Prio {}'.format(data_flow_name_list[i], prio))[-1] + + data_flow.tx_rx.port.tx_name = tx_port_name + data_flow.tx_rx.port.rx_name = rx_port_name + + eth, ipv4, udp = data_flow.packet.ethernet().ipv4().udp() + + eth.src.value = tx_mac + eth.dst.value = rx_mac + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] + + global UDP_PORT_START + src_port = UDP_PORT_START + UDP_PORT_START += 1 + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + + ipv4.src.value = tx_port_config.ip + ipv4.dst.value = rx_port_config.ip + ipv4.priority.choice = ipv4.priority.DSCP + ipv4.priority.dscp.phb.values = prio_dscp_map[prio] + ipv4.priority.dscp.ecn.value = ( + ipv4.priority.dscp.ecn.CAPABLE_TRANSPORT_1) + + data_flow.size.fixed = data_pkt_size + data_flow.rate.percentage = data_flow_rate_percent + data_flow.duration.fixed_seconds.seconds = ( + data_flow_dur_sec_list[i]) + data_flow.duration.fixed_seconds.delay.nanoseconds = int( + sec_to_nanosec(data_flow_delay_sec_list[i])) + + data_flow.metrics.enable = True + data_flow.metrics.loss = True def __run_traffic(api, config, duthost, port, all_flow_names, pfcwd_start_delay_sec, exp_dur_sec): @@ -194,7 +211,7 @@ def __run_traffic(api, config, duthost, port, all_flow_names, pfcwd_start_delay_ """ api.set_config(config) logger.info('Wait for Arp to Resolve ...') - wait_for_arp(api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(api, max_attempts=30, poll_interval_sec=2) logger.info('Starting transmit on all flows ...') ts = api.transmit_state() diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py index 97c5b97958e..0115dbb7117 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py @@ -17,7 +17,7 @@ pytestmark = [pytest.mark.topology('multidut-tgen', 'tgen')] -@pytest.mark.parametrize("trigger_pfcwd", [False]) +@pytest.mark.parametrize("trigger_pfcwd", [True, False]) @pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_multidut_pfcwd_all_to_all(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py index 26b2d0828e3..212169924bf 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py @@ -17,7 +17,7 @@ pytestmark = [pytest.mark.topology('multidut-tgen', 'tgen')] -@pytest.mark.parametrize("trigger_pfcwd", [True]) +@pytest.mark.parametrize("trigger_pfcwd", [True, False]) @pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_many_to_one(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 diff --git a/tests/snappi_tests/test_multidut_snappi.py b/tests/snappi_tests/test_multidut_snappi.py index 0fd2cc78272..1063cc4b971 100644 --- a/tests/snappi_tests/test_multidut_snappi.py +++ b/tests/snappi_tests/test_multidut_snappi.py @@ -149,7 +149,7 @@ def test_snappi(request, snappi_api.set_config(config) # """Wait for Arp""" - wait_for_arp(snappi_api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(snappi_api, max_attempts=30, poll_interval_sec=2) # """ Start traffic """ ts = snappi_api.transmit_state()