diff --git a/tests/common/cisco_data.py b/tests/common/cisco_data.py index f2bc56ea1aa..b0fc51af978 100644 --- a/tests/common/cisco_data.py +++ b/tests/common/cisco_data.py @@ -1,6 +1,7 @@ import json import re from tests.common.reboot import reboot +from tests.common.utilities import wait_until def is_cisco_device(dut): @@ -97,3 +98,18 @@ def set_voq_watchdog(enable): '''.format(enable) copy_dshell_script_cisco_8000(dut, asic, dshell_script, script_name="set_voq_watchdog.py") + + +def check_dshell_ready(duthost): + show_command = "sudo show platform npu rx cgm_global" + err_msg = "debug shell server for asic 0 is not running" + output = duthost.command(show_command)['stdout'] + if err_msg in output: + return False + return True + + +def run_dshell_command(duthost, command): + if not wait_until(300, 20, 0, check_dshell_ready, duthost): + raise RuntimeError("Debug shell is not ready on {}".format(duthost.hostname)) + return duthost.shell(command) diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index 7ff1dcfd86a..c59c7cf836f 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -1921,6 +1921,12 @@ qos/test_ecn_config.py: conditions: - "asic_type not in ['cisco-8000']" +qos/test_oq_watchdog.py: + skip: + reason: "OQ watchdog tests only apply to cisco 8000 platforms." + conditions: + - "asic_type not in ['cisco-8000']" + qos/test_pfc_pause.py::test_pfc_pause_lossless: # For this test, we use the fanout connected to the DUT to send PFC pause frames. # The fanout needs to send PFC frames fast enough so that the queue remains completely paused for the entire duration diff --git a/tests/qos/qos_sai_base.py b/tests/qos/qos_sai_base.py index 8bea5f1e412..51119933357 100644 --- a/tests/qos/qos_sai_base.py +++ b/tests/qos/qos_sai_base.py @@ -18,7 +18,7 @@ from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor from tests.common.mellanox_data import is_mellanox_device as isMellanoxDevice from tests.common.cisco_data import is_cisco_device, copy_set_voq_watchdog_script_cisco_8000, \ - copy_dshell_script_cisco_8000 + copy_dshell_script_cisco_8000, run_dshell_command from tests.common.dualtor.dual_tor_common import active_standby_ports # noqa F401 from tests.common.dualtor.dual_tor_utils import upper_tor_host, lower_tor_host, dualtor_ports, is_tunnel_qos_remap_enabled # noqa F401 from tests.common.dualtor.mux_simulator_control \ @@ -2782,6 +2782,26 @@ def set_port_cir(interface, rate): copy_dshell_script_cisco_8000(dut, asic, dshell_script, script_name="set_scheduler.py") + def get_port_channel_members(self, dut, port_name): + """ + Get the members of portchannel on the given port. + Args: + dut (AnsibleHost): Device Under Test (DUT) + port_name (str): Name of the port to check + Returns: + list: List of port names that are members of the same portchannel. + """ + interfaces = [port_name] + mgfacts = dut.get_extended_minigraph_facts() + for portchan in mgfacts['minigraph_portchannels']: + members = mgfacts['minigraph_portchannels'][portchan]['members'] + if port_name in members: + logger.info("Interface {} is a member of portchannel {}, setting interface list to members {}".format( + port_name, portchan, members)) + interfaces = members + break + return interfaces + @pytest.fixture(scope="function", autouse=False) def set_cir_change(self, get_src_dst_asic_and_duts, dutConfig): dst_port = dutConfig['dutInterfaces'][dutConfig["testPorts"]["dst_port_id"]] @@ -2793,16 +2813,7 @@ def set_cir_change(self, get_src_dst_asic_and_duts, dutConfig): yield return - interfaces = [dst_port] - # If interface is a PortChannel member, expand the list of interfaces that need scheduler setting - mgfacts = dst_dut.get_extended_minigraph_facts() - for portchan in mgfacts['minigraph_portchannels']: - members = mgfacts['minigraph_portchannels'][portchan]['members'] - if dst_port in members: - logger.info("Interface {} is a member of portchannel {}, setting interfaces list to members {}".format( - dst_port, portchan, members)) - interfaces = members - break + interfaces = self.get_port_channel_members(dst_dut, dst_port) # Set scheduler to 5 Gbps. self.copy_set_cir_script_cisco_8000( @@ -2816,11 +2827,11 @@ def set_cir_change(self, get_src_dst_asic_and_duts, dutConfig): def voq_watchdog_enabled(self, get_src_dst_asic_and_duts): dst_dut = get_src_dst_asic_and_duts['dst_dut'] - if dst_dut.facts['asic_type'] != "cisco-8000": + if not is_cisco_device(dst_dut): return False namespace_option = "-n asic0" if dst_dut.facts.get("modular_chassis") else "" show_command = "show platform npu global {}".format(namespace_option) - result = dst_dut.command(show_command) + result = run_dshell_command(dst_dut, show_command) pattern = r"voq_watchdog_enabled +: +True" match = re.search(pattern, result["stdout"]) return match @@ -2882,3 +2893,64 @@ def disable_voq_watchdog_function_scope(self, duthosts, get_src_dst_asic_and_dut def disable_voq_watchdog_class_scope(self, duthosts, get_src_dst_asic_and_duts, dutConfig): with self.disable_voq_watchdog(duthosts, get_src_dst_asic_and_duts, dutConfig) as result: yield result + + def oq_watchdog_enabled(self, get_src_dst_asic_and_duts): + dst_dut = get_src_dst_asic_and_duts['dst_dut'] + if not is_cisco_device(dst_dut): + return False + namespace_option = "-n asic0" if dst_dut.facts.get("modular_chassis") else "" + show_command = "show platform npu global {}".format(namespace_option) + result = run_dshell_command(dst_dut, show_command) + pattern = r"gb_watchdog_oq_enabled +: +True" + match = re.search(pattern, result["stdout"]) + return match + + def copy_set_queue_pir_script_cisco_8000(self, dut, ports, queue, speed, schduler_type, asic=""): + dshell_script = ''' +from common import * +def set_queue_pir(interface, queue, rate): + sai_lane = port_to_sai_lane_map[interface] + slice_id, ifg_id, serdes_id = sai_lane_to_slice_ifg_serdes(sai_lane) + sys_port = find_system_port(slice_id, ifg_id, serdes_id) + scheduler = sys_port.get_scheduler() + original_pir = scheduler.get_{}_pir(queue) + print(original_pir) + scheduler.set_{}_pir(queue, rate) +'''.format(schduler_type, schduler_type) + + for intf in ports: + dshell_script += 'set_queue_pir("{}", {}, {})\n'.format(intf, queue, speed) + + copy_dshell_script_cisco_8000(dut, asic, dshell_script, script_name="set_queue_pir.py") + + def block_queue(self, dut, port, queue, queue_type, asic_index=""): + interfaces = self.get_port_channel_members(dut, port) + scheduler_type = "credit" if queue_type == "voq" else "transmit" + self.copy_set_queue_pir_script_cisco_8000( + dut=dut, + ports=interfaces, + queue=queue, + asic=asic_index, + speed=0, + schduler_type=scheduler_type) + cmd_opt = "-n asic{}".format(asic_index) + if not dut.sonichost.is_multi_asic: + cmd_opt = "" + result = dut.shell("sudo show platform npu script {} -s set_queue_pir.py".format(cmd_opt)) + original_pir = result["stdout_lines"][0] + return int(original_pir) + + def unblock_queue(self, dut, port, queue, queue_type, speed, asic_index=""): + interfaces = self.get_port_channel_members(dut, port) + scheduler_type = "credit" if queue_type == "voq" else "transmit" + self.copy_set_queue_pir_script_cisco_8000( + dut=dut, + ports=interfaces, + queue=queue, + asic=asic_index, + speed=speed, + schduler_type=scheduler_type) + cmd_opt = "-n asic{}".format(asic_index) + if not dut.sonichost.is_multi_asic: + cmd_opt = "" + dut.shell("sudo show platform npu script {} -s set_queue_pir.py".format(cmd_opt)) diff --git a/tests/qos/test_oq_watchdog.py b/tests/qos/test_oq_watchdog.py new file mode 100644 index 00000000000..ada1ab498f0 --- /dev/null +++ b/tests/qos/test_oq_watchdog.py @@ -0,0 +1,125 @@ +"""SAI thrift-based tests for the OQ watchdog feature in SONiC. + +This set of test cases verifies OQ watchdog behavior. These are dataplane +tests that depend on the SAI thrift library in order to pause ports and read +drop counters. + +Parameters: + --ptf_portmap (str): file name of port index to DUT interface alias map. Default is None. + In case a filename is not provided, a file containing a port indices to aliases map will be generated. + + --qos_swap_syncd (bool): Used to install the RPC syncd image before running the tests. Default is True. + + --qos_dst_ports (list) Indices of available DUT test ports to serve as destination ports. Note: This is not port + index on DUT, rather an index into filtered (excludes lag member ports) DUT ports. Plan is to randomize port + selection. Default is [0, 1, 3]. + + --qos_src_ports (list) Indices of available DUT test ports to serve as source port. Similar note as in + qos_dst_ports applies. Default is [2]. +""" + +import logging +import pytest + +from tests.common.fixtures.duthost_utils import dut_qos_maps, \ + separated_dscp_to_tc_map_on_uplink # noqa F401 +from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import copy_saitests_directory # noqa F401 +from tests.common.fixtures.ptfhost_utils import change_mac_addresses # noqa F401 +from .qos_sai_base import QosSaiBase + +logger = logging.getLogger(__name__) + +pytestmark = [ + pytest.mark.topology('any') +] + +PKTS_NUM = 100 + + +@pytest.fixture(scope="function") +def ignore_log_oq_watchdog(duthosts, loganalyzer): + if not loganalyzer: + yield + return + ignore_list = [r".*HARDWARE_WATCHDOG.*", r".*soft_reset*"] + for dut in duthosts: + for line in ignore_list: + loganalyzer[dut.hostname].ignore_regex.append(line) + yield + return + + +class TestOqWatchdog(QosSaiBase): + """TestVoqWatchdog derives from QosSaiBase and contains collection of OQ watchdog test cases. + """ + @pytest.fixture(scope="class", autouse=True) + def check_skip_oq_watchdog_test(self, get_src_dst_asic_and_duts): + if not self.oq_watchdog_enabled(get_src_dst_asic_and_duts): + pytest.skip("OQ watchdog test is skipped since OQ watchdog is not enabled.") + + def testOqWatchdog( + self, ptfhost, dutTestParams, dutConfig, dutQosConfig, + get_src_dst_asic_and_duts, ignore_log_oq_watchdog, + disable_voq_watchdog_function_scope + ): + """ + Test OQ watchdog functionality. + Test steps: + 1. block voq7, sys_port scheduler set Q7 credit_pir to 0 + 2. fill leakout of Q7 by ping, make sure no packet dequeue/enqueue in OQ7 afterwards + 3. block oq0, sys_port scheduler set Q0 transmit_pir to 0 + 4. send traffic on Q0, oq watchdog should be triggered in about 5 seconds + 5. Unblock voq7 and oq0 to restore the system state + 6. Run TrafficSanityTest to verify the system state is restored + Args: + ptfhost (AnsibleHost): Packet Test Framework (PTF) + dutTestParams (Fixture, dict): DUT host test params + dutConfig (Fixture, dict): Map of DUT config containing dut interfaces, test port IDs, test port IPs, + and test ports + dutQosConfig (Fixture, dict): Map containing DUT host QoS configuration + Returns: + None + Raises: + RunAnsibleModuleFail if ptf test fails + """ + + # Block voq7 + dst_dut = get_src_dst_asic_and_duts['dst_dut'] + dst_asic_index = get_src_dst_asic_and_duts['dst_asic_index'] + dst_port = dutConfig['dutInterfaces'][dutConfig["testPorts"]["dst_port_id"]] + original_pir_voq7 = self.block_queue(dst_dut, dst_port, 7, "voq", dst_asic_index) + # Fill leakout of Q7 by ping + cmd_opt = "sudo ip netns exec asic{}".format(dst_asic_index) + if not dst_dut.sonichost.is_multi_asic: + cmd_opt = "" + dst_dut.shell("{} ping -I {} -c 50 1.1.1.1 -i 0 -w 0 || true".format(cmd_opt, dst_port)) + + # Block oq0 + original_pir_oq0 = self.block_queue(dst_dut, dst_port, 0, "oq", dst_asic_index) + + testParams = dict() + testParams.update(dutTestParams["basicParams"]) + testParams.update({ + "dscp": 8, + "dst_port_id": dutConfig["testPorts"]["dst_port_id"], + "dst_port_ip": dutConfig["testPorts"]["dst_port_ip"], + "src_port_id": dutConfig["testPorts"]["src_port_id"], + "src_port_ip": dutConfig["testPorts"]["src_port_ip"], + "src_port_vlan": dutConfig["testPorts"]["src_port_vlan"], + "packet_size": 1350, + "pkts_num": PKTS_NUM, + "oq_watchdog_enabled": True, + }) + + self.runPtfTest( + ptfhost, testCase="sai_qos_tests.OqWatchdogTest", + testParams=testParams) + + # Unblock voq7 and oq0 to restore the system state + self.unblock_queue(dst_dut, dst_port, 7, "voq", original_pir_voq7, dst_asic_index) + self.unblock_queue(dst_dut, dst_port, 0, "oq", original_pir_oq0, dst_asic_index) + + self.runPtfTest( + ptfhost, testCase="sai_qos_tests.TrafficSanityTest", + testParams=testParams) diff --git a/tests/qos/test_voq_watchdog.py b/tests/qos/test_voq_watchdog.py index 1a18231b997..69c75b562e8 100644 --- a/tests/qos/test_voq_watchdog.py +++ b/tests/qos/test_voq_watchdog.py @@ -94,6 +94,10 @@ def testVoqWatchdog( ptfhost, testCase="sai_qos_tests.VoqWatchdogTest", testParams=testParams) + self.runPtfTest( + ptfhost, testCase="sai_qos_tests.TrafficSanityTest", + testParams=testParams) + def testVoqWatchdogDisable( self, ptfhost, dutTestParams, dutConfig, dutQosConfig, get_src_dst_asic_and_duts, disable_voq_watchdog_function_scope @@ -129,3 +133,7 @@ def testVoqWatchdogDisable( self.runPtfTest( ptfhost, testCase="sai_qos_tests.VoqWatchdogTest", testParams=testParams) + + self.runPtfTest( + ptfhost, testCase="sai_qos_tests.TrafficSanityTest", + testParams=testParams) diff --git a/tests/saitests/py3/sai_qos_tests.py b/tests/saitests/py3/sai_qos_tests.py index 5cd749caf7e..82592e464db 100755 --- a/tests/saitests/py3/sai_qos_tests.py +++ b/tests/saitests/py3/sai_qos_tests.py @@ -125,10 +125,13 @@ DEFAULT_PKT_COUNT = 10 PG_TOLERANCE = 2 -# Constants for voq watchdog test -VOQ_WATCHDOG_TIMEOUT_SECONDS = 60 -SAI_LOG_TO_CHECK = ["HARDWARE_WATCHDOG", "soft_reset"] -SDK_LOG_TO_CHECK = ["VOQ Appears to be stuck"] +# Constants for voq/oq watchdog test +WATCHDOG_TIMEOUT_SECONDS = {"voq": 60, + "oq": 5} +SAI_LOG_TO_CHECK = {"voq": ["HARDWARE_WATCHDOG", "soft_reset"], + "oq": ["HARDWARE_WATCHDOG", "soft_reset"]} +SDK_LOG_TO_CHECK = {"voq": ["VOQ Appears to be stuck"], + "oq": []} SAI_LOG = "/var/log/sai.log" SDK_LOG = "/var/log/syslog" @@ -678,6 +681,46 @@ def get_peer_addr(data, addr): return list(addresses) +def init_log_check(test_case): + pre_offsets = [] + for logfile in [SAI_LOG, SDK_LOG]: + offset_cmd = "stat -c %s {}".format(logfile) + stdout, err, ret = test_case.exec_cmd_on_dut( + test_case.dst_server_ip, + test_case.test_params['dut_username'], + test_case.test_params['dut_password'], + offset_cmd) + pre_offsets.append(int(stdout[0])) + return pre_offsets + + +def verify_log(test_case, pre_offsets, watchdog_enabled=True, watchdog_type='voq'): + qos_test_assert(test_case, watchdog_type in ['voq', 'oq'], + "Invalid watchdog type: {}".format(watchdog_type)) + found_list = [] + for pre_offset, logfile, str_to_check in zip(pre_offsets, [SAI_LOG, SDK_LOG], + [SAI_LOG_TO_CHECK[watchdog_type], SDK_LOG_TO_CHECK[watchdog_type]]): + egrep_str = '|'.join(str_to_check) + check_cmd = "sudo tail -c +{} {} | egrep '{}' || true".format(pre_offset + 1, logfile, egrep_str) + stdout, err, ret = test_case.exec_cmd_on_dut( + test_case.dst_server_ip, + test_case.test_params['dut_username'], + test_case.test_params['dut_password'], + check_cmd) + log_message("Log for {}: {}".format(egrep_str, stdout)) + for string in str_to_check: + if string in "".join(stdout): + found_list.append(True) + else: + found_list.append(False) + if watchdog_enabled: + qos_test_assert(test_case, all(found is True for found in found_list), + "{} watchdog trigger not detected".format(watchdog_type)) + else: + qos_test_assert(test_case, all(found is False for found in found_list), + "unexpected {} watchdog trigger".format(watchdog_type)) + + class ARPpopulate(sai_base_test.ThriftInterfaceDataPlane): def setUp(self): sai_base_test.ThriftInterfaceDataPlane.setUp(self) @@ -6632,42 +6675,6 @@ def runTest(self): class VoqWatchdogTest(sai_base_test.ThriftInterfaceDataPlane): - def init_log_check(self): - pre_offsets = [] - for logfile in [SAI_LOG, SDK_LOG]: - offset_cmd = "stat -c %s {}".format(logfile) - stdout, err, ret = self.exec_cmd_on_dut( - self.dst_server_ip, - self.test_params['dut_username'], - self.test_params['dut_password'], - offset_cmd) - pre_offsets.append(int(stdout[0])) - return pre_offsets - - def verify_log(self, pre_offsets, voq_watchdog_enabled=True): - found_list = [] - for pre_offset, logfile, str_to_check in zip(pre_offsets, [SAI_LOG, SDK_LOG], - [SAI_LOG_TO_CHECK, SDK_LOG_TO_CHECK]): - egrep_str = '|'.join(str_to_check) - check_cmd = "sudo tail -c +{} {} | egrep '{}' || true".format(pre_offset + 1, logfile, egrep_str) - stdout, err, ret = self.exec_cmd_on_dut( - self.dst_server_ip, - self.test_params['dut_username'], - self.test_params['dut_password'], - check_cmd) - log_message("Log for {}: {}".format(egrep_str, stdout)) - for string in str_to_check: - if string in "".join(stdout): - found_list.append(True) - else: - found_list.append(False) - if voq_watchdog_enabled: - qos_test_assert(self, all(found is True for found in found_list), - "VOQ watchdog trigger not detected") - else: - qos_test_assert(self, all(found is False for found in found_list), - "unexpected VOQ watchdog trigger") - def runTest(self): switch_init(self.clients) @@ -6721,49 +6728,170 @@ def runTest(self): log_message("actual dst_port_id: {}".format(dst_port_id), to_stderr=True) self.sai_thrift_port_tx_disable(self.dst_client, asic_type, [dst_port_id]) - pre_offsets = self.init_log_check() + pre_offsets = init_log_check(self) try: # send packets send_packet(self, src_port_id, pkt, pkts_num) # allow enough time to trigger voq watchdog - time.sleep(VOQ_WATCHDOG_TIMEOUT_SECONDS * 1.3) + time.sleep(WATCHDOG_TIMEOUT_SECONDS["voq"] * 1.3) # verify voq watchdog is triggered - self.verify_log(pre_offsets, voq_watchdog_enabled) + verify_log(self, pre_offsets, voq_watchdog_enabled, "voq") + finally: self.sai_thrift_port_tx_enable(self.dst_client, asic_type, [dst_port_id]) + +class OqWatchdogTest(sai_base_test.ThriftInterfaceDataPlane): + def runTest(self): + switch_init(self.clients) + + # Parse input parameters + dscp = int(self.test_params['dscp']) + router_mac = self.test_params['router_mac'] + sonic_version = self.test_params['sonic_version'] + dst_port_id = int(self.test_params['dst_port_id']) + dst_port_ip = self.test_params['dst_port_ip'] + dst_port_mac = self.dataplane.get_mac(0, dst_port_id) + src_port_id = int(self.test_params['src_port_id']) + src_port_ip = self.test_params['src_port_ip'] + src_port_vlan = self.test_params['src_port_vlan'] + src_port_mac = self.dataplane.get_mac(0, src_port_id) + oq_watchdog_enabled = self.test_params['oq_watchdog_enabled'] + pkts_num = int(self.test_params['pkts_num']) + + pkt_dst_mac = router_mac if router_mac != '' else dst_port_mac + # get counter names to query + ingress_counters, egress_counters = get_counter_names(sonic_version) + + # Prepare IP packet data + ttl = 64 + if 'packet_size' in list(self.test_params.keys()): + packet_length = int(self.test_params['packet_size']) + else: + packet_length = 64 + + is_dualtor = self.test_params.get('is_dualtor', False) + def_vlan_mac = self.test_params.get('def_vlan_mac', None) + if is_dualtor and def_vlan_mac is not None: + pkt_dst_mac = def_vlan_mac + + pkt = construct_ip_pkt(packet_length, + pkt_dst_mac, + src_port_mac, + src_port_ip, + dst_port_ip, + dscp, + src_port_vlan, + ttl=ttl) + + log_message("test dst_port_id: {}, src_port_id: {}, src_vlan: {}".format( + dst_port_id, src_port_id, src_port_vlan), to_stderr=True) + + pre_offsets = init_log_check(self) + + try: + # send packets + send_packet(self, src_port_id, pkt, pkts_num) + + # allow enough time to trigger oq watchdog + time.sleep(WATCHDOG_TIMEOUT_SECONDS["oq"] * 1.3) + + # verify voq watchdog is triggered + verify_log(self, pre_offsets, oq_watchdog_enabled, "oq") + + finally: + print("END OF TEST") + + +class TrafficSanityTest(sai_base_test.ThriftInterfaceDataPlane): + def runTest(self): + switch_init(self.clients) + + # Parse input parameters + dscp = int(self.test_params['dscp']) + router_mac = self.test_params['router_mac'] + sonic_version = self.test_params['sonic_version'] + dst_port_id = int(self.test_params['dst_port_id']) + dst_port_ip = self.test_params['dst_port_ip'] + dst_port_mac = self.dataplane.get_mac(0, dst_port_id) + src_port_id = int(self.test_params['src_port_id']) + src_port_ip = self.test_params['src_port_ip'] + src_port_vlan = self.test_params['src_port_vlan'] + src_port_mac = self.dataplane.get_mac(0, src_port_id) + asic_type = self.test_params['sonic_asic_type'] + pkts_num = int(self.test_params['pkts_num']) + exp_ip_id = 110 + + pkt_dst_mac = router_mac if router_mac != '' else dst_port_mac + # get counter names to query + ingress_counters, egress_counters = get_counter_names(sonic_version) + + # Prepare IP packet data + ttl = 64 + if 'packet_size' in list(self.test_params.keys()): + packet_length = int(self.test_params['packet_size']) + else: + packet_length = 64 + + is_dualtor = self.test_params.get('is_dualtor', False) + def_vlan_mac = self.test_params.get('def_vlan_mac', None) + if is_dualtor and def_vlan_mac is not None: + pkt_dst_mac = def_vlan_mac + + pkt = construct_ip_pkt(packet_length, + pkt_dst_mac, + src_port_mac, + src_port_ip, + dst_port_ip, + dscp, + src_port_vlan, + ttl=ttl) + + log_message("test dst_port_id: {}, src_port_id: {}, src_vlan: {}".format( + dst_port_id, src_port_id, src_port_vlan), to_stderr=True) + # in case dst_port_id is part of LAG, find out the actual dst port + # for given IP parameters + dst_port_id = get_rx_port( + self, 0, src_port_id, pkt_dst_mac, dst_port_ip, src_port_ip, src_port_vlan + ) + log_message("actual dst_port_id: {}".format(dst_port_id), to_stderr=True) + + try: # allow enough time for the dut to sync up the counter values in counters_db time.sleep(8) # get a snapshot of counter values at recv and transmit ports recv_counters_base, _ = sai_thrift_read_port_counters( self.src_client, asic_type, port_list['src'][src_port_id]) - xmit_counters_base, queue_counters_base = sai_thrift_read_port_counters( + xmit_counters_base, _ = sai_thrift_read_port_counters( self.dst_client, asic_type, port_list['dst'][dst_port_id]) - if voq_watchdog_enabled: - # queue counters should be cleared after soft reset - qos_test_assert( - self, queue_counters_base[0] == 0, - 'queue counters are not cleared, soft reset is not triggered') # send packets + pkt = construct_ip_pkt(packet_length, + pkt_dst_mac, + src_port_mac, + src_port_ip, + dst_port_ip, + dscp, + src_port_vlan, + ip_id=exp_ip_id, + ttl=ttl) send_packet(self, src_port_id, pkt, pkts_num) + # allow enough time for the dut to sync up the counter values in counters_db time.sleep(8) - # get a snapshot of counter values at recv and transmit ports recv_counters, _ = sai_thrift_read_port_counters( self.src_client, asic_type, port_list['src'][src_port_id]) - xmit_counters, queue_counters = sai_thrift_read_port_counters( + xmit_counters, _ = sai_thrift_read_port_counters( self.dst_client, asic_type, port_list['dst'][dst_port_id]) log_message( '\trecv_counters {}\n\trecv_counters_base {}\n\t' - 'xmit_counters {}\n\txmit_counters_base {}\n\t' - 'queue_counters {}\n\tqueue_counters_base {}\n'.format( - recv_counters, recv_counters_base, xmit_counters, xmit_counters_base, - queue_counters, queue_counters_base), to_stderr=True) + 'xmit_counters {}\n\txmit_counters_base {}\n'.format( + recv_counters, recv_counters_base, xmit_counters, xmit_counters_base), + to_stderr=True) # recv port no ingress drop for cntr in ingress_counters: qos_test_assert( @@ -6774,11 +6902,34 @@ def runTest(self): qos_test_assert( self, xmit_counters[cntr] == xmit_counters_base[cntr], 'unexpectedly TX drop counter increase') - # queue counters increased by pkts_num - qos_test_assert( - self, queue_counters[0] == queue_counters_base[0] + pkts_num, - 'queue counter not matched, expected {}, got {}'.format( - queue_counters_base[0] + pkts_num, queue_counters[0])) + + # Set receiving socket buffers to some big value + for p in list(self.dataplane.ports.values()): + p.socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 41943040) + + cnt = 0 + recv_pkt = scapy.Ether() + while recv_pkt: + received = self.dataplane.poll( + device_number=0, port_number=dst_port_id, timeout=2) + if isinstance(received, self.dataplane.PollFailure): + recv_pkt = None + break + recv_pkt = scapy.Ether(received.packet) + + try: + if recv_pkt[scapy.IP].src == src_port_ip and recv_pkt[scapy.IP].dst == dst_port_ip and \ + recv_pkt[scapy.IP].id == exp_ip_id: + cnt += 1 + except AttributeError: + continue + except IndexError: + # Ignore captured non-IP packet + continue + + # All packets sent should be received intact + qos_test_assert(self, pkts_num == cnt, + 'Received {} packets, expected {}'.format(cnt, pkts_num)) finally: - self.sai_thrift_port_tx_enable(self.dst_client, asic_type, [dst_port_id]) + print("END OF TEST")