From 09d085d39d8a09eaca069fa8558bc9ada9542580 Mon Sep 17 00:00:00 2001 From: Sonic Build Admin Date: Wed, 29 Jan 2025 05:56:03 +0000 Subject: [PATCH] [T0 T1 merge] update multidut pfcwd tests Summary: Update the missing T0 T1 tests feature in multidut Fixes # (issue) 30929279 The following PR were migrated for pfcwd - https://github.com/sonic-net/sonic-mgmt/pull/7370 - https://github.com/sonic-net/sonic-mgmt/pull/9567 - https://github.com/sonic-net/sonic-mgmt/pull/7367 to multidut code - https://github.com/sonic-net/sonic-mgmt/pull/7712 ### Type of change - [ ] Bug fix - [ ] Testbed and Framework(new/improvement) - [ ] New Test case - [ ] Skipped for non-supported platforms - [ ] Test case improvement ### Back port request - [ ] 202012 - [ ] 202205 - [ ] 202305 - [ ] 202311 - [ ] 202405 - [ ] 202411 ### Approach #### What is the motivation for this PR? #### How did you do it? #### How did you verify/test it? #### Any platform specific information? #### Supported testbed topology if it's a new test case? ### Documentation snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py ``` -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html - generated xml file: /var/src/sonic-mgmt_vmsvc5-t2-8800-ixia_669de75fbbfea63e8b73b319/tests/logs/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.xml - ================== 4 passed, 7 warnings in 1235.72s (0:20:35) ================== ``` snappi_tests/multidut/pfcwd/test_multidut_pfcwd_burst_storm_with_snappi.py ``` -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html - generated xml file: /var/src/sonic-mgmt_vmsvc5-t2-8800-ixia_669de75fbbfea63e8b73b319/tests/logs/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_burst_storm_with_snappi.xml - ================== 2 passed, 5 warnings in 858.60s (0:14:18) =================== ``` snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py (known failure on T2, fail as expected) ``` SKIPPED [4] snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py:190: Reboot type fast is not supported on cisco-8000 switches FAILED snappi_tests/multidut/pfcwd/test_multidut_pfcwd_basic_with_snappi.py::test_pfcwd_basic_multi_lossless_prio_restart_service[multidut_port_info0-True-swss] ===== 1 failed, 23 passed, 16 skipped, 27 warnings in 27779.49s (7:42:59) ====== ``` snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py ``` -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html - generated xml file: /var/src/sonic-mgmt_vmsvc5-t2-8800-ixia_669de75fbbfea63e8b73b319/tests/logs/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.xml - ================== 4 passed, 7 warnings in 1157.40s (0:19:17) ================== ``` snappi_tests/multidut/pfcwd/test_multidut_pfcwd_runtime_traffic_with_snappi.py ``` -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html - generated xml file: /var/src/sonic-mgmt_vmsvc5-t2-8800-ixia_669de75fbbfea63e8b73b319/tests/logs/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_runtime_traffic_with_snappi.xml - ================== 2 passed, 6 warnings in 1162.35s (0:19:22) ================== ``` --- .../pfcwd_multidut_burst_storm_helper.py | 4 +- .../files/pfcwd_multidut_multi_node_helper.py | 45 ++++--- .../pfcwd_multidut_runtime_traffic_helper.py | 113 ++++++++++-------- .../test_multidut_pfcwd_a2a_with_snappi.py | 2 +- .../test_multidut_pfcwd_m2o_with_snappi.py | 2 +- tests/snappi_tests/test_multidut_snappi.py | 2 +- 6 files changed, 100 insertions(+), 68 deletions(-) diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py index daa2048a2a..db5c80922d 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_burst_storm_helper.py @@ -247,7 +247,7 @@ def __gen_traffic(testbed_config, pause_pkt.pause_class_6.value = pause_time[6] pause_pkt.pause_class_7.value = pause_time[7] - pause_flow_start_time = id * (pause_flow_dur_sec + pause_flow_gap_sec) + pause_flow_start_time = id * (pause_flow_dur_sec + pause_flow_gap_sec) + WARM_UP_TRAFFIC_DUR pause_flow.rate.pps = pause_pps pause_flow.size.fixed = 64 @@ -275,7 +275,7 @@ def __run_traffic(api, config, all_flow_names, exp_dur_sec): api.set_config(config) logger.info('Wait for Arp to Resolve ...') - wait_for_arp(api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(api, max_attempts=30, poll_interval_sec=2) logger.info('Starting transmit on all flows ...') ts = api.transmit_state() diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py index 18e07532fb..7a03706956 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_multi_node_helper.py @@ -153,6 +153,11 @@ def run_pfcwd_multi_node_test(api, all_flow_names=all_flow_names, exp_dur_sec=exp_dur_sec) + """ Retrieve ASIC information for DUT """ + asic_type = egress_duthost.facts['asic_type'] + + rx_tx_tol_thrhlds = [0.0001, 0.0002] # Maintain a 0.01% and 0.02% deviation between tx and rx frames + __verify_results(rows=flow_stats, speed_gbps=speed_gbps, pause_flow_name=PAUSE_FLOW_NAME, @@ -164,7 +169,9 @@ def run_pfcwd_multi_node_test(api, data_pkt_size=DATA_PKT_SIZE, trigger_pfcwd=trigger_pfcwd, pause_port_id=rx_port_id_list[0], - tolerance=TOLERANCE_THRESHOLD) + rx_deviation=TOLERANCE_THRESHOLD, + rx_tx_deviations=rx_tx_tol_thrhlds, + asic_type=asic_type) def __data_flow_name(name_prefix, src_id, dst_id, prio): @@ -532,7 +539,7 @@ def __run_traffic(api, config, all_flow_names, exp_dur_sec): api.set_config(config) logger.info('Wait for Arp to Resolve ...') - wait_for_arp(api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(api, max_attempts=30, poll_interval_sec=2) logger.info('Starting transmit on all flows ...') ts = api.transmit_state() @@ -586,7 +593,9 @@ def __verify_results(rows, data_pkt_size, trigger_pfcwd, pause_port_id, - tolerance): + rx_deviation, + rx_tx_deviations, + asic_type): """ Verify if we get expected experiment results @@ -602,11 +611,16 @@ def __verify_results(rows, test_flow_pause (bool): if test flows are expected to be paused trigger_pfcwd (bool): if PFC watchdog is expected to be triggered pause_port_id (int): ID of the port to send PFC pause frames - tolerance (float): maximum allowable deviation + rx_deviation (float): maximum allowable deviation for rx_frames relative to theoretical value + rx_tx_deviations (list of floats): maximum allowable % deviation for rx_frames relative to tx_frames Returns: N/A """ + + """ Check for whether DUT is a Mellanox device """ + is_mlnx_device = True if "mellanox" in asic_type.lower() else False + for row in rows: flow_name = row.name tx_frames = row.frames_tx @@ -629,7 +643,7 @@ def __verify_results(rows, exp_bg_flow_rx_pkts = bg_flow_rate_percent / 100.0 * speed_gbps \ * 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size deviation = (rx_frames - exp_bg_flow_rx_pkts) / float(exp_bg_flow_rx_pkts) - pytest_assert(abs(deviation) < tolerance, + pytest_assert(abs(deviation) < rx_deviation, '{} should receive {} packets (actual {})'. format(flow_name, exp_bg_flow_rx_pkts, rx_frames)) @@ -641,14 +655,17 @@ def __verify_results(rows, exp_test_flow_rx_pkts = test_flow_rate_percent / 100.0 * speed_gbps \ * 1e9 * data_flow_dur_sec / 8.0 / data_pkt_size - if trigger_pfcwd and\ - (src_port_id == pause_port_id or dst_port_id == pause_port_id): + if trigger_pfcwd and dst_port_id == pause_port_id: """ Once PFC watchdog is triggered, it will impact bi-directional traffic """ logger.info('Once PFC watchdog is triggered, it will impact bi-directional traffic') logger.info('Tx and Rx should have dropped packets') pytest_assert(tx_frames > rx_frames, '{} should have dropped packets'.format(flow_name)) - + elif trigger_pfcwd and src_port_id == pause_port_id: + if is_mlnx_device: + """ During a pfc storm with pfcwd triggered, Mellanox devices do not drop Rx packets """ + pytest_assert(tx_frames == rx_frames, + '{} should not have dropped packets for Mellanox device'.format(flow_name)) elif not trigger_pfcwd and dst_port_id == pause_port_id: """ This test flow is delayed by PFC storm """ logger.info('This test flow is delayed by PFC storm') @@ -660,14 +677,12 @@ def __verify_results(rows, format(flow_name, exp_test_flow_rx_pkts, rx_frames)) else: - """ Otherwise, the test flow is not impacted by PFC storm """ - logger.info('the test flow is not impacted by PFC storm') - logger.info('Tx and Rx should not have any dropped packet') - - pytest_assert(tx_frames == rx_frames, - '{} should not have any dropped packet'.format(flow_name)) + for dev_pct in rx_tx_deviations: + """ Otherwise, the test flow is not impacted by PFC storm """ + pytest_assert(abs(tx_frames - rx_frames)/float(tx_frames) < dev_pct, + '{} should be within {} percent deviation'.format(flow_name, dev_pct*100)) deviation = (rx_frames - exp_test_flow_rx_pkts) / float(exp_test_flow_rx_pkts) - pytest_assert(abs(deviation) < tolerance, + pytest_assert(abs(deviation) < rx_deviation, '{} should receive {} packets (actual {})'. format(flow_name, exp_test_flow_rx_pkts, rx_frames)) diff --git a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py index f1d92a421b..e88fb75502 100644 --- a/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py +++ b/tests/snappi_tests/multidut/pfcwd/files/pfcwd_multidut_runtime_traffic_helper.py @@ -3,16 +3,18 @@ from tests.common.helpers.assertions import pytest_assert from tests.common.snappi_tests.snappi_helpers import get_dut_port_id # noqa: F401 -from tests.common.snappi_tests.common_helpers import start_pfcwd, stop_pfcwd +from tests.common.snappi_tests.common_helpers import start_pfcwd, stop_pfcwd, sec_to_nanosec from tests.common.snappi_tests.port import select_ports, select_tx_port # noqa: F401 from tests.common.snappi_tests.snappi_helpers import wait_for_arp from tests.common.snappi_tests.snappi_test_params import SnappiTestParams from tests.snappi_tests.variables import pfcQueueGroupSize, pfcQueueValueDict DATA_FLOW_NAME = "Data Flow" +WARM_UP_TRAFFIC_NAME = "Warm Up Traffic" DATA_PKT_SIZE = 1024 DATA_FLOW_DURATION_SEC = 15 -PFCWD_START_DELAY_SEC = 3 +WARM_UP_TRAFFIC_DUR = 1 +PFCWD_START_DELAY_SEC = 3 + WARM_UP_TRAFFIC_DUR SNAPPI_POLL_DELAY_SEC = 2 TOLERANCE_THRESHOLD = 0.05 UDP_PORT_START = 5000 @@ -65,12 +67,20 @@ def run_pfcwd_runtime_traffic_test(api, stop_pfcwd(egress_duthost, rx_port['asic_value']) stop_pfcwd(ingress_duthost, tx_port['asic_value']) + """ Warm up traffic is initially sent before any other traffic to prevent pfcwd + fake alerts caused by idle links (non-incremented packet counters) during pfcwd detection periods """ + warm_up_traffic_dur_sec = WARM_UP_TRAFFIC_DUR + warm_up_traffic_delay_sec = 0 + __gen_traffic(testbed_config=testbed_config, port_config_list=port_config_list, tx_port_id=tx_port_id, rx_port_id=rx_port_id, - data_flow_name=DATA_FLOW_NAME, - data_flow_dur_sec=DATA_FLOW_DURATION_SEC, + data_flow_name_list=[WARM_UP_TRAFFIC_NAME, DATA_FLOW_NAME], + data_flow_delay_sec_list=[ + warm_up_traffic_delay_sec, WARM_UP_TRAFFIC_DUR], + data_flow_dur_sec_list=[ + warm_up_traffic_dur_sec, DATA_FLOW_DURATION_SEC], data_pkt_size=DATA_PKT_SIZE, prio_list=prio_list, prio_dscp_map=prio_dscp_map) @@ -90,7 +100,8 @@ def run_pfcwd_runtime_traffic_test(api, speed_str = testbed_config.layer1[0].speed speed_gbps = int(speed_str.split('_')[1]) - __verify_results(rows=flow_stats, + data_flows = [flow_stat for flow_stat in flow_stats if DATA_FLOW_NAME in flow_stat.name] + __verify_results(rows=data_flows, speed_gbps=speed_gbps, data_flow_dur_sec=DATA_FLOW_DURATION_SEC, data_pkt_size=DATA_PKT_SIZE, @@ -101,8 +112,9 @@ def __gen_traffic(testbed_config, port_config_list, tx_port_id, rx_port_id, - data_flow_name, - data_flow_dur_sec, + data_flow_name_list, + data_flow_delay_sec_list, + data_flow_dur_sec_list, data_pkt_size, prio_list, prio_dscp_map): @@ -113,8 +125,9 @@ def __gen_traffic(testbed_config, testbed_config (obj): testbed L1/L2/L3 configuration port_config_list (list): list of port configuration port_id (int): ID of DUT port to test. - data_flow_name (str): data flow name - data_flow_dur_sec (int): duration of data flows in second + data_flow_name_list (list): list of data flow names + data_flow_delay_sec_list (list): list of data flow start delays in second + data_flow_dur_sec_list (list): list of data flow durations in second data_pkt_size (int): size of data packets in byte prio_list (list): priorities of data flows prio_dscp_map (dict): Priority vs. DSCP map (key = priority). @@ -136,44 +149,48 @@ def __gen_traffic(testbed_config, tx_port_name = testbed_config.ports[tx_port_id].name rx_port_name = testbed_config.ports[rx_port_id].name data_flow_rate_percent = int(100 / len(prio_list)) - - """ For each priority """ - for prio in prio_list: - data_flow = testbed_config.flows.flow( - name='{} Prio {}'.format(data_flow_name, prio))[-1] - - data_flow.tx_rx.port.tx_name = tx_port_name - data_flow.tx_rx.port.rx_name = rx_port_name - - eth, ipv4, udp = data_flow.packet.ethernet().ipv4().udp() - - eth.src.value = tx_mac - eth.dst.value = rx_mac - if pfcQueueGroupSize == 8: - eth.pfc_queue.value = prio - else: - eth.pfc_queue.value = pfcQueueValueDict[prio] - - global UDP_PORT_START - src_port = UDP_PORT_START - UDP_PORT_START += 1 - udp.src_port.increment.start = src_port - udp.src_port.increment.step = 1 - udp.src_port.increment.count = 1 - - ipv4.src.value = tx_port_config.ip - ipv4.dst.value = rx_port_config.ip - ipv4.priority.choice = ipv4.priority.DSCP - ipv4.priority.dscp.phb.values = prio_dscp_map[prio] - # ipv4.priority.dscp.ecn.value = ( - # ipv4.priority.dscp.ecn.CAPABLE_TRANSPORT_1) - - data_flow.size.fixed = data_pkt_size - data_flow.rate.percentage = data_flow_rate_percent - data_flow.duration.fixed_seconds.seconds = data_flow_dur_sec - - data_flow.metrics.enable = True - data_flow.metrics.loss = True + """ For each data flow """ + for i in range(len(data_flow_name_list)): + """ For each priority """ + for prio in prio_list: + data_flow = testbed_config.flows.flow( + name='{} Prio {}'.format(data_flow_name_list[i], prio))[-1] + + data_flow.tx_rx.port.tx_name = tx_port_name + data_flow.tx_rx.port.rx_name = rx_port_name + + eth, ipv4, udp = data_flow.packet.ethernet().ipv4().udp() + + eth.src.value = tx_mac + eth.dst.value = rx_mac + if pfcQueueGroupSize == 8: + eth.pfc_queue.value = prio + else: + eth.pfc_queue.value = pfcQueueValueDict[prio] + + global UDP_PORT_START + src_port = UDP_PORT_START + UDP_PORT_START += 1 + udp.src_port.increment.start = src_port + udp.src_port.increment.step = 1 + udp.src_port.increment.count = 1 + + ipv4.src.value = tx_port_config.ip + ipv4.dst.value = rx_port_config.ip + ipv4.priority.choice = ipv4.priority.DSCP + ipv4.priority.dscp.phb.values = prio_dscp_map[prio] + ipv4.priority.dscp.ecn.value = ( + ipv4.priority.dscp.ecn.CAPABLE_TRANSPORT_1) + + data_flow.size.fixed = data_pkt_size + data_flow.rate.percentage = data_flow_rate_percent + data_flow.duration.fixed_seconds.seconds = ( + data_flow_dur_sec_list[i]) + data_flow.duration.fixed_seconds.delay.nanoseconds = int( + sec_to_nanosec(data_flow_delay_sec_list[i])) + + data_flow.metrics.enable = True + data_flow.metrics.loss = True def __run_traffic(api, config, duthost, port, all_flow_names, pfcwd_start_delay_sec, exp_dur_sec): @@ -194,7 +211,7 @@ def __run_traffic(api, config, duthost, port, all_flow_names, pfcwd_start_delay_ """ api.set_config(config) logger.info('Wait for Arp to Resolve ...') - wait_for_arp(api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(api, max_attempts=30, poll_interval_sec=2) logger.info('Starting transmit on all flows ...') ts = api.transmit_state() diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py index 97c5b97958..0115dbb711 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_a2a_with_snappi.py @@ -17,7 +17,7 @@ pytestmark = [pytest.mark.topology('multidut-tgen', 'tgen')] -@pytest.mark.parametrize("trigger_pfcwd", [False]) +@pytest.mark.parametrize("trigger_pfcwd", [True, False]) @pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_multidut_pfcwd_all_to_all(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 diff --git a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py index 26b2d0828e..212169924b 100644 --- a/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py +++ b/tests/snappi_tests/multidut/pfcwd/test_multidut_pfcwd_m2o_with_snappi.py @@ -17,7 +17,7 @@ pytestmark = [pytest.mark.topology('multidut-tgen', 'tgen')] -@pytest.mark.parametrize("trigger_pfcwd", [True]) +@pytest.mark.parametrize("trigger_pfcwd", [True, False]) @pytest.mark.parametrize("multidut_port_info", MULTIDUT_PORT_INFO[MULTIDUT_TESTBED]) def test_pfcwd_many_to_one(snappi_api, # noqa: F811 conn_graph_facts, # noqa: F811 diff --git a/tests/snappi_tests/test_multidut_snappi.py b/tests/snappi_tests/test_multidut_snappi.py index 0fd2cc7827..1063cc4b97 100644 --- a/tests/snappi_tests/test_multidut_snappi.py +++ b/tests/snappi_tests/test_multidut_snappi.py @@ -149,7 +149,7 @@ def test_snappi(request, snappi_api.set_config(config) # """Wait for Arp""" - wait_for_arp(snappi_api, max_attempts=10, poll_interval_sec=2) + wait_for_arp(snappi_api, max_attempts=30, poll_interval_sec=2) # """ Start traffic """ ts = snappi_api.transmit_state()