From caad974d416cca8bb110227eb6c8b1643f657e1b Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Wed, 2 Feb 2022 03:33:30 +0000 Subject: [PATCH 01/11] Adding all testcases, and the required modifications. --- .../test/files/ptftests/vxlan_traffic.py | 51 ++- tests/vxlan/test_vxlan_ecmp.py | 424 +++++++++++++++++- 2 files changed, 457 insertions(+), 18 deletions(-) diff --git a/ansible/roles/test/files/ptftests/vxlan_traffic.py b/ansible/roles/test/files/ptftests/vxlan_traffic.py index 4345cb8b5a2..853c3904ab5 100644 --- a/ansible/roles/test/files/ptftests/vxlan_traffic.py +++ b/ansible/roles/test/files/ptftests/vxlan_traffic.py @@ -1,5 +1,5 @@ # ptf --test-dir ptftests vxlan_traffic.VXLAN --platform-dir ptftests --qlen=1000 --platform remote \ -# -t 't2_ports=[16, 17, 0, 1, 4, 5, 21, 20];dut_mac=u"64:3a:ea:c1:73:f8";expect_encap_success=True; \ +# -t 't2_ports=[16, 17, 0, 1, 4, 5, 21, 20];dut_mac=u"64:3a:ea:c1:73:f8";expect_encap_success=True;packet_count=10; \ # vxlan_port=4789;topo_file="/tmp/vxlan_topo_file.json";config_file="/tmp/vxlan-config-TC1-v6_in_v4.json";t0_ports=[u"Ethernet42"]' --relax --debug info \ # --log-file /tmp/vxlan-tests.TC1.v6_in_v4.log @@ -36,7 +36,10 @@ def get_incremental_value(key): global VARS - VARS[key] = VARS[key] + 1 + VARS[key] = (VARS[key] + 1) % 65535 + # We would like to use the ports from 1234 to 65535 + if VARS[key] < 1234: + VARS[key] = 1234 return VARS[key] def read_ptf_macs(): @@ -59,6 +62,12 @@ def setUp(self): self.dut_mac = self.test_params['dut_mac'] self.vxlan_port = self.test_params['vxlan_port'] self.expect_encap_success = self.test_params['expect_encap_success'] + self.packet_count = self.test_params['packet_count'] + # The ECMP check fails occasionally if there is not enough packets. + # We should keep the packet count atleast 4. + if self.packet_count < 4: + print("WARN: packet_count is below minimum, resetting to 4") + self.packet_count = 4 self.random_mac = "00:aa:bb:cc:dd:ee" self.ptf_mac_addrs = read_ptf_macs() @@ -139,12 +148,11 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, else: tagged = False - - options = {'ip_tos' : 0} - options_v6 = {'ipv6_tc' : 0} + options = {'ip_ecn' : 0} + options_v6 = {'ipv6_ecn' : 0} if test_ecn: - options = {'ip_tos' : random.randint(0, 3)} - options_v6 = {'ipv6_tos' : random.randint(0, 3)} + options = {'ip_ecn' : random.randint(0, 3)} + options_v6 = {'ipv6_ecn' : random.randint(0, 3)} # ECMP support, assume it is a string of comma seperated list of addresses. returned_ip_addresses = {} @@ -152,7 +160,7 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, for host_address in nhs: check_ecmp = True # This will ensure that every nh is used atleast once. - for i in range(4): + for i in range(self.packet_count): tcp_sport = get_incremental_value('tcp_sport') tcp_dport = 5000 valid_combination = True @@ -190,7 +198,6 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, exp_pkt = simple_tcpv6_packet(**pkt_opts) else: valid_combination = False - print("Unusable combination:src:{} and dst:{}".format(src, destination)) udp_sport = 1234 # Use entropy_hash(pkt), it will be ignored in the test later. udp_dport = self.vxlan_port if isinstance(ip_address(host_address), ipaddress.IPv4Address): @@ -205,7 +212,8 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, udp_dport=udp_dport, with_udp_chksum=False, vxlan_vni=vni, - inner_frame=exp_pkt) + inner_frame=exp_pkt, + **options) encap_pkt[IP].flags = 0x2 elif isinstance(ip_address(host_address), ipaddress.IPv6Address): encap_pkt = simple_vxlanv6_packet( @@ -217,8 +225,9 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, udp_dport=udp_dport, with_udp_chksum=False, vxlan_vni=vni, - inner_frame=exp_pkt) - send_packet(self, ptf_port, str(pkt), count=2) + inner_frame=exp_pkt, + **options_v6) + send_packet(self, ptf_port, str(pkt)) masked_exp_pkt = Mask(encap_pkt) masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "src") @@ -256,8 +265,24 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, # Verify ECMP: if check_ecmp: + # Check #1 : All addresses have been used. if set(nhs) - set(returned_ip_addresses.keys()) == set([]): - print ("Each address has been used") + print (" Each address has been used") + print ("Packets sent:{} distribution:".format(self.packet_count)) + for nh_address in returned_ip_addresses.keys(): + print (" {} : {}".format(nh_address, returned_ip_addresses[nh_address])) + # Check #2 : The packets are almost equally distributed. + # We calculate the total number of packets sent, and average it across the number of ECMP hops. + # Every next-hop should have received within 1% of the packets that we sent per nexthop. + # This check is valid only if there are large enough number of packets. + if self.packet_count > 300: + tolerance = 0.01 + for nh_address in returned_ip_addresses.keys(): + if ((1.0-tolerance) * self.packet_count > returned_ip_addresses[nh_address] + or (1.0+tolerance) * self.packet_count < returned_ip_addresses[nh_address]): + raise RuntimeError("ECMP nexthop address: {} received too less or too many of the " + "packets expected. Expected:{}, received on that address:{}".format(nh_address, self.packet_count, returned_ip_addresses[nh_address])) + else: raise RuntimeError('''ECMP might have failed for:{}, we expected every ip address in the nexthop group({} of them) to be used, but only {} are used:\nUsed addresses:{}\nUnused Addresses:{}'''.format(destination, diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index 7490a9dbaa9..7d4f5d856a7 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -38,6 +38,8 @@ import pytest from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory # lgtm[py/unused-import] +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until from tests.ptf_runner import ptf_runner Logger = logging.getLogger(__name__) @@ -60,6 +62,10 @@ # This list is used in many locations in the script. SUPPORTED_ENCAP_TYPES = ['v4_in_v4', 'v4_in_v6', 'v6_in_v4', 'v6_in_v6'] +# Starting prefixes to be used for the destinations and End points. +Destination_Prefix = 150 +NextHop_Prefix = 100 + pytestmark = [ # This script supports any T1 topology: t1, t1-64-lag, t1-lag. pytest.mark.topology("t1"), @@ -430,7 +436,7 @@ def set_routes_in_dut(duthost, dest_to_nh_map, dest_af, op): config_list.append(create_single_route(vnet, dest, HOST_MASK[dest_af], dest_to_nh_map[vnet][dest], op)) full_config = '[' + "\n,".join(config_list) + '\n]' - apply_config_in_swss(duthost, full_config, "set_routes") + apply_config_in_swss(duthost, full_config, op+"_routes") def get_t2_ports(duthost, minigraph_data): ''' @@ -549,8 +555,8 @@ def setUp(duthosts, ptfhost, request, rand_one_dut_hostname, minigraph_facts, number_of_available_nexthops=request.config.option.total_number_of_endpoints, number_of_ecmp_nhs=request.config.option.total_number_of_nexthops, dest_af=payload_version, - dest_net_prefix=150, # Hardcoded to avoid conflicts with topology networks. - nexthop_prefix=100, # Hardcoded to avoid conflicts with topology networks. + dest_net_prefix=Destination_Prefix, # Hardcoded to avoid conflicts with topology networks. + nexthop_prefix=NextHop_Prefix, # Hardcoded to avoid conflicts with topology networks. nh_af=outer_layer_version) data[encap_type] = encap_type_data @@ -589,7 +595,7 @@ def setUp(duthosts, ptfhost, request, rand_one_dut_hostname, minigraph_facts, @pytest.mark.parametrize("encap_type", SUPPORTED_ENCAP_TYPES) class Test_VxLAN: - def dump_self_info_and_run_ptf(self, tcname, encap_type, expect_encap_success): + def dump_self_info_and_run_ptf(self, tcname, encap_type, expect_encap_success, packet_count=4): ''' Just a wrapper for dump_info_to_ptf to avoid entering 30 lines everytime. ''' @@ -620,7 +626,8 @@ def dump_self_info_and_run_ptf(self, tcname, encap_type, expect_encap_success): "t2_ports":self.setup[encap_type]['t2_ports'], "dut_mac":self.setup['dut_mac'], "vxlan_port": self.setup['vxlan_port'], - "expect_encap_success":expect_encap_success + "expect_encap_success":expect_encap_success, + "packet_count":packet_count }, qlen=1000, log_file="/tmp/vxlan-tests.{}.{}.{}.log".format(tcname, encap_type, datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))) @@ -630,3 +637,410 @@ def test_vxlan_single_endpoint(self, setUp, encap_type): self.setup = setUp Logger.info("tc1:Create a tunnel route to a single endpoint a. Send packets to the route prefix dst.") self.dump_self_info_and_run_ptf("tc1", encap_type, True) + + def test_vxlan_modify_route_different_endpoint(self, setUp, request, encap_type): + ######################################################################################### + # testcae 2: change the route to different endpoint. + ######################################################################################### + self.setup = setUp + Logger.info("tc2: change the route to different endpoint. packets are received only at endpoint b.") + + # Choose a vnet + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Choose a destination, which is already present. + tc2_dest = self.setup[encap_type]['dest_to_nh_map'][vnet].keys()[0] + + # Create a new endpoint, or endpoint-list. + tc2_new_end_point_list = [] + for i in range(int(request.config.option.ecmp_nhs_per_destination)): + tc2_new_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + + # Map the destination to the new endpoint(s). + self.setup[encap_type]['dest_to_nh_map'][vnet][tc2_dest] = tc2_new_end_point_list + + # Create the json and apply the config in the DUT swss. + # The config looks like: + # [ + # { + # "VNET_ROUTE_TUNNEL_TABLE:vnet:tc2_dest/32": { + # "endpoint": "{tc2_new_end_point_list}" + # }, + # "OP": "{}" + # } + # ] + tc2_full_config = '[\n' + create_single_route(vnet, tc2_dest, HOST_MASK[get_payload_version(encap_type)], tc2_new_end_point_list, "SET") + '\n]' + apply_config_in_swss(self.setup['duthost'], tc2_full_config, "vnet_route_tc2_"+encap_type) + + # Copy the new set of configs to the PTF and run the tests. + self.dump_self_info_and_run_ptf("tc2", encap_type, True) + + @pytest.mark.skip(reason="causes syncd restarts and failures of later tests.") + def test_vxlan_remove_all_route(self, setUp, encap_type): + Logger.info("tc3: remove the tunnel route. send packets to the route prefix dst. packets should not be received at any ports with dst ip of b") + self.setup = setUp + try: + # Remove the existing routes in the DUT. + set_routes_in_dut(self.setup['duthost'], self.setup[encap_type]['dest_to_nh_map'], get_payload_version(encap_type), "DEL") + # Verify that the traffic is not coming back. + self.dump_self_info_and_run_ptf("tc3", encap_type, False) + finally: + # Restore the routes in the DUT. + set_routes_in_dut(self.setup['duthost'], self.setup[encap_type]['dest_to_nh_map'], get_payload_version(encap_type), "SET") + +class Test_VxLAN_ecmp_create(Test_VxLAN): + def test_vxlan_configure_route1_ecmp_group_a(self, setUp, encap_type): + self.setup = setUp + Logger.info("tc4:create tunnel route 1 with two endpoints a = {a1, a2...}. send packets to the route 1's prefix dst. packets are received at either a1 or a2") + + # Choose a vnet. + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Create a new list of endpoint(s). + tc4_end_point_list = [] + for i in range(2): + tc4_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + + # Create a new destination + tc4_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=Destination_Prefix) + + # Map the new destination and the new endpoint(s). + self.setup[encap_type]['dest_to_nh_map'][vnet][tc4_new_dest] = tc4_end_point_list + + # Create a new config and Copy to the DUT. + tc4_config = '[\n' + create_single_route(vnet, tc4_new_dest, HOST_MASK[get_payload_version(encap_type)], tc4_end_point_list, "SET") + '\n]' + apply_config_in_swss(self.setup['duthost'], tc4_config, "vnet_route_tc4_"+encap_type) + + # Verify that the new config takes effect and runs traffic. + self.dump_self_info_and_run_ptf("tc4", encap_type, True) + + def test_vxlan_configure_route1_ecmp_group_b(self, setUp, encap_type): + self.setup = setUp + Logger.info('tc5: set tunnel route 2 to endpoint group a = {a1, a2}. send packets to route 2"s prefix dst. packets are received at either a1 or a2') + self.setup_route2_ecmp_group_b(encap_type) + # Verify the configs work and traffic flows correctly. + self.dump_self_info_and_run_ptf("tc5", encap_type, True) + + def setup_route2_ecmp_group_b(self, encap_type): + if self.setup[encap_type].get('tc5_dest', None): + return + # Choose a vnet for testing. + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Select an existing endpoint. + tc5_end_point_list = self.setup[encap_type]['dest_to_nh_map'][vnet].values()[0] + + # Create a new destination to use. + tc5_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=Destination_Prefix) + + # Map the new destination to the endpoint. + self.setup[encap_type]['dest_to_nh_map'][vnet][tc5_new_dest] = tc5_end_point_list + + # Create the new config and apply to the DUT. + tc5_config = '[\n' + create_single_route(vnet, tc5_new_dest, HOST_MASK[get_payload_version(encap_type)], tc5_end_point_list, "SET") + '\n]' + apply_config_in_swss(self.setup['duthost'], tc5_config, "vnet_route_tc5_"+encap_type) + self.setup[encap_type]['tc5_dest'] = tc5_new_dest + + def test_vxlan_configure_route2_ecmp_group_b(self, setUp, encap_type): + self.setup = setUp + self.setup_route2_ecmp_group_b(encap_type) + Logger.info('tc6: set tunnel route 2 to endpoint group b = {b1, b2}. send packets to route 2"s prefix dst. packets are received at either b1 or b2') + + # Choose a vnet for testing. + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Create a new list of endpoints. + tc6_end_point_list = [] + for i in range(2): + tc6_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + + # Choose one of the existing destinations. + tc6_new_dest = self.setup[encap_type]['tc5_dest'] + + # Map the destination to the new endpoints. + self.setup[encap_type]['dest_to_nh_map'][vnet][tc6_new_dest] = tc6_end_point_list + + # Crete the config and apply on the DUT. + tc6_config = '[\n' + create_single_route(vnet, tc6_new_dest, HOST_MASK[get_payload_version(encap_type)], tc6_end_point_list, "SET") + '\n]' + apply_config_in_swss(self.setup['duthost'], tc6_config, "vnet_route_tc6_"+encap_type) + + # Verify that the traffic works. + self.dump_self_info_and_run_ptf("tc6", encap_type, True) + +class Test_VxLAN_NHG_Modify(Test_VxLAN): + + def setup_route2_single_endpoint(self, encap_type): + if self.setup[encap_type].get('tc8_dest', None): + return + + # Pick a vnet for testing. + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Choose a route 2 destination and a new single endpoint for it. + tc8_new_dest = self.setup[encap_type]['dest_to_nh_map'][vnet].keys()[0] + tc8_new_nh = get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix) + Logger.info("Using destinations: dest:{} => nh:{}".format(tc8_new_dest, tc8_new_nh)) + + # Map the destination and new endpoint. + tc8_config = '[\n' + create_single_route(vnet, tc8_new_dest, HOST_MASK[get_payload_version(encap_type)], [tc8_new_nh], "SET") + '\n]' + self.setup[encap_type]['dest_to_nh_map'][vnet][tc8_new_dest] = [tc8_new_nh] + + # Apply the new config in the DUT and run traffic test. + apply_config_in_swss(self.setup['duthost'], tc8_config, "vnet_route_tc8_"+encap_type) + self.setup[encap_type]['tc8_dest'] = tc8_new_dest + + def setup_route2_shared_endpoints(self, encap_type): + if self.setup[encap_type].get('tc9_dest', None): + return + self.setup_route2_single_endpoint(encap_type) + + # Choose a vnet for testing. + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Select 2 already existing destinations. + tc9_new_dest1 = self.setup[encap_type]['tc8_dest'] + + tc9_new_dest2 = self.setup[encap_type]['dest_to_nh_map'][vnet].keys()[0] + Logger.info("Using destinations: dest1:{}, dest2:{}".format(tc9_new_dest1, tc9_new_dest2)) + + # Combine the first of their tunnel endpoints. + tc9_new_nhs = [self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1][0]] + [self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest2][0]] + + # Map the destination 1 to the combined list. + self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1] = tc9_new_nhs + tc9_config = '[\n' + create_single_route(vnet, tc9_new_dest1, HOST_MASK[get_payload_version(encap_type)], tc9_new_nhs, "SET") + '\n]' + + # Apply the new config to the DUT and send traffic. + apply_config_in_swss(self.setup['duthost'], tc9_config, "vnet_route_tc9_"+encap_type) + self.setup[encap_type]['tc9_dest'] = tc9_new_dest1 + + def test_vxlan_remove_route2(self, setUp, encap_type): + self.setup = setUp + Logger.info("tc7:send packets to route 1's prefix dst. by removing route 2 from group a, no change expected to route 1.") + + # Pick a vnet for testing. + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Setup: Create two destinations with the same endpoint group. + tc7_end_point_list = [] + for i in range(2): + tc7_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + + tc7_destinations = [] + for i in range(2): + tc7_destinations.append(get_ip_address(af=get_payload_version(encap_type), netid=Destination_Prefix)) + + # Map the new destinations to the same endpoint list. + for i in range(2): + self.setup[encap_type]['dest_to_nh_map'][vnet][tc7_destinations[i]] = tc7_end_point_list + + # Apply the setup configs to the DUT. + for i in range(2): + tc7_setup_config = '[\n' + create_single_route(vnet, tc7_destinations[i], HOST_MASK[get_payload_version(encap_type)], tc7_end_point_list, "SET") + '\n]' + apply_config_in_swss(self.setup['duthost'], tc7_setup_config, "vnet_route_tc7_"+encap_type) + + # verify the setup works. + self.dump_self_info_and_run_ptf("tc7", encap_type, True) + # End of setup. + + # now remove one of the routes. + # Pick one out of the two TC7 destinations. + tc7_removed_dest = tc7_destinations[0] + tc7_removed_endpoint = self.setup[encap_type]['dest_to_nh_map'][vnet][tc7_removed_dest] + del self.setup[encap_type]['dest_to_nh_map'][vnet][tc7_removed_dest] + + # Remove the chosen dest/endpoint from the DUT. + tc7_config = '[\n' + create_single_route(vnet, tc7_removed_dest, HOST_MASK[get_payload_version(encap_type)], tc7_removed_endpoint, "DEL") + '\n]' + apply_config_in_swss(self.setup['duthost'], tc7_config, "vnet_route_tc7_"+encap_type) + + # Verify the rest of the traffic still works. + self.dump_self_info_and_run_ptf("tc7", encap_type, True) + + def test_vxlan_route2_single_nh(self, setUp, encap_type): + self.setup = setUp + Logger.info("tc8: set tunnel route 2 to single endpoint b1. send packets to route 2's prefix dst") + self.setup_route2_single_endpoint(encap_type) + self.dump_self_info_and_run_ptf("tc8", encap_type, True) + + def test_vxlan_route2_shared_nh(self, setUp, encap_type): + self.setup = setUp + Logger.info("tc9: set tunnel route 2 to shared endpoints a1 and b1. send packets to route 2's prefix dst") + self.setup_route2_shared_endpoints(encap_type) + self.dump_self_info_and_run_ptf("tc9", encap_type, True) + + def test_vxlan_remove_ecmp_route2(self, setUp, encap_type): + self.setup = setUp + self.setup_route2_shared_endpoints(encap_type) + # backup the current route config. + full_map = dict(self.setup[encap_type]['dest_to_nh_map']) + + Logger.info("tc10: remove tunnel route 2. send packets to route 2's prefix dst") + + # This is to keep track if the selected route should be deleted in the end. + del_needed = False + try: + # Choose a vnet for testing. + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Choose a destination and its nhs to delete. + tc10_dest = self.setup[encap_type]['tc9_dest'] + tc10_nhs = self.setup[encap_type]['dest_to_nh_map'][vnet][tc10_dest] + Logger.info("Using destination: dest:{}, nh:{}".format(tc10_dest, tc10_nhs)) + + # Delete the dest and nh in the DUT. + tc10_config = '[\n' + create_single_route(vnet, tc10_dest, HOST_MASK[get_payload_version(encap_type)], tc10_nhs, "DEL") + '\n]' + apply_config_in_swss(self.setup['duthost'], tc10_config, "vnet_route_tc10_"+encap_type) + del_needed = True + + # We should pass only the deleted entry to the ptf call, and expect encap to fail. + # Clear out the mappings, and keep only the deleted dest and nhs. + self.setup[encap_type]['dest_to_nh_map'][vnet] = {} + self.setup[encap_type]['dest_to_nh_map'][vnet][tc10_dest] = tc10_nhs + + # the deleted route should fail to receive traffic. + self.dump_self_info_and_run_ptf("tc10", encap_type, False) + + # all others should be working. + # Housekeeping: + # Restore the mapping of dest->nhs. + self.setup[encap_type]['dest_to_nh_map'] = dict(full_map) + # Remove the deleted entry alone. + del self.setup[encap_type]['dest_to_nh_map'][vnet][tc10_dest] + del_needed = False + + # Check the traffic is working in the other routes. + self.dump_self_info_and_run_ptf("tc10", encap_type, True) + + except: + self.setup[encap_type]['dest_to_nh_map'] = dict(full_map) + # Remove the deleted entry alone. + if del_needed: + del self.setup[encap_type]['dest_to_nh_map'][vnet][tc10_dest] + raise + +class Test_VxLAN_ecmp_random_hash(Test_VxLAN): + def test_vxlan_random_hash(self, setUp, encap_type): + self.setup = setUp + Logger.info("tc11: set tunnel route 3 to endpoint group c = {c1, c2, c3}. ensure c1, c2, and c3 matches to underlay default route. send 1000 pkt with random hash to route 3's prefix dst") + + # Chose a vnet for testing. + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + # Create a new destination and 3 nhs for it. + tc11_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=Destination_Prefix) + tc11_new_nhs = [] + for i in range(3): + tc11_new_nhs.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + + # the topology always provides the default routes for any ip address. + # so it is already taken care of. + + # Map the new dest and nhs. + tc11_config = '[\n' + create_single_route(vnet, tc11_new_dest, HOST_MASK[get_payload_version(encap_type)], tc11_new_nhs, "SET") + '\n]' + self.setup[encap_type]['dest_to_nh_map'][vnet][tc11_new_dest] = tc11_new_nhs + + # Apply the config in the DUT and verify traffic. The random hash and ECMP check is already taken care of in the VxLAN PTF script. + apply_config_in_swss(self.setup['duthost'], tc11_config, "vnet_route_tc11_"+encap_type) + self.dump_self_info_and_run_ptf("tc11", encap_type, True, packet_count=1000) + +@pytest.mark.skip(reason="All cases fail due to syncd crash and docker container restarts.") +class Test_VxLAN_underlay_ecmp(Test_VxLAN): + @pytest.mark.parametrize("ecmp_path_count", [1, 2]) + def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, ecmp_path_count): + self.setup = setUp + Logger.info("tc12: modify the underlay default route nexthop/s. send packets to route 3's prefix dst.") + # First step: pick one or two of the interfaces connected to t2, and bring them down. + # verify that the encap is still working, and ptf receives the traffic. + # Bring them back up. + # After that, bring down all the other t2 interfaces, other than the ones used in the first step. + # This will force a modification to the underlay default routes nexthops. + + all_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) + if not all_intfs: + all_intfs = get_ethernet_to_neighbors("T2", minigraph_facts) + Logger.info("Dumping T2 link info: {}".format(all_intfs)) + if not all_intfs: + raise RuntimeError("no interface found connected to t2 neighbors. pls check the testbed, aborting.") + + # Backup + all_t2_ports = list(self.setup[encap_type]['t2_ports']) + try: + shut_intfs = [] + # Choose some intfs based on the parameter ecmp_path_count. + # when ecmp_path_count == 1, it is non-ecmp. The switching happens between ecmp and non-ecmp. + # Otherwise, the switching happens within ecmp only. + for i in range(ecmp_path_count): + shut_intfs.append(all_intfs[i]) + + for intf in shut_intfs: + self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) + self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) + + # Bring up the selected intfs. + + self.setup[encap_type]['t2_ports'] = list(all_t2_ports) + for intf in shut_intfs: + self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) + time.sleep(60) + # Shutdown other interfaces. + for intf in set(all_intfs) - set(shut_intfs): + self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) + # allow time for recovery. + time.sleep(60) + self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) + + # Recover all interfaces. + self.setup[encap_type]['t2_ports'] = list(all_t2_ports) + for intf in all_intfs: + self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) + + # Wait for all bgp is up. + pytest_assert(wait_until(300, 30, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") + + # Verify traffic flows after recovery. + self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) + + finally: + # If anything goes wrong in the try block, atleast bring the intf back up. + self.setup[encap_type]['t2_ports'] = list(all_t2_ports) + for intf in all_intfs: + self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) + pytest_assert(wait_until(300, 30, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") + + def test_vxlan_remove_add_underlay_default(self, setUp, minigraph_facts, encap_type): + self.setup = setUp + Logger.info("tc13: remove the underlay default route.") + + # Find all the underlay default routes' interfaces. This means all T2 interfaces. + all_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) + if not all_intfs: + all_intfs = get_ethernet_to_neighbors("T2", minigraph_facts) + Logger.info("Dumping T2 link info: {}".format(all_intfs)) + if not all_intfs: + raise RuntimeError("no interface found connected to t2 neighbors. pls check the testbed, aborting.") + + try: + # Bring down the T2 interfaces. + for intf in all_intfs: + self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) + + # Verify that traffic is not flowing through. + self.dump_self_info_and_run_ptf("tc12", encap_type, False) + + Logger.info("tc14: Re-add the underlay default route.") + + # Bring up the T2 interfaces. + for intf in all_intfs: + self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) + + # Wait for all bgp is up. + pytest_assert(wait_until(300, 30, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") + + # Verify the traffic is flowing through, again. + self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) + + finally: + # If anything goes wrong in the try block, atleast bring the intf back up. + for intf in all_intfs: + self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) + pytest_assert(wait_until(300, 30, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") From 962c002a4db2b5c3f3026831104f01108642ad33 Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Wed, 2 Feb 2022 05:12:27 +0000 Subject: [PATCH 02/11] Removed the skips. --- tests/vxlan/test_vxlan_ecmp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index 7d4f5d856a7..c060df6f34a 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -675,7 +675,6 @@ def test_vxlan_modify_route_different_endpoint(self, setUp, request, encap_type) # Copy the new set of configs to the PTF and run the tests. self.dump_self_info_and_run_ptf("tc2", encap_type, True) - @pytest.mark.skip(reason="causes syncd restarts and failures of later tests.") def test_vxlan_remove_all_route(self, setUp, encap_type): Logger.info("tc3: remove the tunnel route. send packets to the route prefix dst. packets should not be received at any ports with dst ip of b") self.setup = setUp @@ -943,7 +942,6 @@ def test_vxlan_random_hash(self, setUp, encap_type): apply_config_in_swss(self.setup['duthost'], tc11_config, "vnet_route_tc11_"+encap_type) self.dump_self_info_and_run_ptf("tc11", encap_type, True, packet_count=1000) -@pytest.mark.skip(reason="All cases fail due to syncd crash and docker container restarts.") class Test_VxLAN_underlay_ecmp(Test_VxLAN): @pytest.mark.parametrize("ecmp_path_count", [1, 2]) def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, ecmp_path_count): From 2c47f9df4fb300df7af7041d2e69a15f36232135 Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Tue, 15 Mar 2022 00:35:25 +0000 Subject: [PATCH 03/11] Completed review comments. --- .../test/files/ptftests/vxlan_traffic.py | 92 ++++---- tests/vxlan/test_vxlan_ecmp.py | 223 ++++++++++++------ 2 files changed, 201 insertions(+), 114 deletions(-) diff --git a/ansible/roles/test/files/ptftests/vxlan_traffic.py b/ansible/roles/test/files/ptftests/vxlan_traffic.py index 853c3904ab5..47f981650e5 100644 --- a/ansible/roles/test/files/ptftests/vxlan_traffic.py +++ b/ansible/roles/test/files/ptftests/vxlan_traffic.py @@ -17,29 +17,33 @@ import ptf.packet as scapy from ptf.base_tests import BaseTest from ptf import config -from ptf.testutils import * +from ptf.testutils import (simple_tcp_packet, simple_tcpv6_packet, simple_vxlan_packet, simple_vxlanv6_packet, + verify_packet_any_port, verify_no_packet_any, + send_packet, test_params_get) from ptf.dataplane import match_exp_pkt from ptf.mask import Mask import datetime import subprocess import ipaddress -from pprint import pprint +import logging from ipaddress import ip_address import random + VARS = {} VARS['tcp_sport'] = 1234 VARS['tcp_dport'] = 5000 +logger = logging.getLogger(__name__) + # Some constants used in this code +MIN_PACKET_COUNT = 4 +MINIMUM_PACKETS_FOR_ECMP_VALIDATION = 300 TEST_ECN = False def get_incremental_value(key): - global VARS - VARS[key] = (VARS[key] + 1) % 65535 # We would like to use the ports from 1234 to 65535 - if VARS[key] < 1234: - VARS[key] = 1234 + VARS[key] = max(1234, (VARS[key] + 1) % 65535) return VARS[key] def read_ptf_macs(): @@ -64,10 +68,10 @@ def setUp(self): self.expect_encap_success = self.test_params['expect_encap_success'] self.packet_count = self.test_params['packet_count'] # The ECMP check fails occasionally if there is not enough packets. - # We should keep the packet count atleast 4. - if self.packet_count < 4: - print("WARN: packet_count is below minimum, resetting to 4") - self.packet_count = 4 + # We should keep the packet count atleast MIN_PACKET_COUNT. + if self.packet_count < MIN_PACKET_COUNT: + logger.warn("Packet_count is below minimum, resetting to {}", MIN_PACKET_COUNT) + self.packet_count = MIN_PACKET_COUNT self.random_mac = "00:aa:bb:cc:dd:ee" self.ptf_mac_addrs = read_ptf_macs() @@ -138,6 +142,32 @@ def read_ptf_macs(self): return addrs + def verify_all_addresses_used_equally(self, nhs, returned_ip_addresses): + ''' + Verify the ECMP functionality using 2 checks. + Check 1 verifies every nexthop address has been used. + Check 2 verifies the distribution of number of packets among the nexthops. + Params: nhs: the nexthops that are configured. + returned_ip_addresses: The dict containing the nh addresses and corresponding packet counts. + ''' + # Check #1 : All addresses have been used. + if set(nhs) - set(returned_ip_addresses.keys()) == set([]): + logger.info (" Each address has been used") + logger.info ("Packets sent:{} distribution:".format(self.packet_count)) + for nh_address in returned_ip_addresses.keys(): + logger.info (" {} : {}".format(nh_address, returned_ip_addresses[nh_address])) + # Check #2 : The packets are almost equally distributed. + # Every next-hop should have received within 1% of the packets that we sent per nexthop(which is self.packet_count). + # This check is valid only if there are large enough number of packets(300). Any lower number will need higher tolerance(more than 2%). + if self.packet_count > MINIMUM_PACKETS_FOR_ECMP_VALIDATION: + tolerance = 0.01 + for nh_address in returned_ip_addresses.keys(): + if (1.0-tolerance) * self.packet_count <= returned_ip_addresses[nh_address] <= (1.0+tolerance) * self.packet_count: + pass + else: + raise RuntimeError("ECMP nexthop address: {} received too less or too many of the " + "packets expected. Expected:{}, received on that address:{}".format(nh_address, self.packet_count, returned_ip_addresses[nh_address])) + def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, vlan=0): rv = True try: @@ -162,7 +192,6 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, # This will ensure that every nh is used atleast once. for i in range(self.packet_count): tcp_sport = get_incremental_value('tcp_sport') - tcp_dport = 5000 valid_combination = True if isinstance(ip_address(destination), ipaddress.IPv4Address) and isinstance(ip_address(ptf_addr), ipaddress.IPv4Address): pkt_opts = { @@ -174,7 +203,7 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, "ip_id":105, "ip_ttl":64, "tcp_sport":tcp_sport, - "tcp_dport":tcp_dport} + "tcp_dport":VARS['tcp_dport']} pkt_opts.update(options) pkt = simple_tcp_packet(**pkt_opts) pkt_opts['ip_ttl'] = 63 @@ -189,7 +218,7 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, "ipv6_src":ptf_addr, "ipv6_hlim":64, "tcp_sport":tcp_sport, - "tcp_dport":tcp_dport} + "tcp_dport":VARS['tcp_dport']} pkt_opts.update(options_v6) pkt = simple_tcpv6_packet(**pkt_opts) pkt_opts['ipv6_hlim'] = 63 @@ -214,7 +243,7 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, vxlan_vni=vni, inner_frame=exp_pkt, **options) - encap_pkt[IP].flags = 0x2 + encap_pkt[scapy.IP].flags = 0x2 elif isinstance(ip_address(host_address), ipaddress.IPv6Address): encap_pkt = simple_vxlanv6_packet( eth_src=self.dut_mac, @@ -243,11 +272,11 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, masked_exp_pkt.set_do_not_care_scapy(scapy.UDP, "sport") masked_exp_pkt.set_do_not_care_scapy(scapy.UDP, "chksum") - logging.info("Sending packet from port " + str(ptf_port) + " to " + destination) + logger.info("Sending packet from port " + str(ptf_port) + " to " + destination) if self.expect_encap_success: - status, received_pkt = verify_packet_any_port(self, masked_exp_pkt, self.t2_ports) - scapy_pkt = Ether(received_pkt) + _, received_pkt = verify_packet_any_port(self, masked_exp_pkt, self.t2_ports) + scapy_pkt = scapy.Ether(received_pkt) # Store every destination that was received. if isinstance(ip_address(host_address), ipaddress.IPv6Address): dest_ip = scapy_pkt['IPv6'].dst @@ -260,36 +289,15 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, else: check_ecmp = False - print ("Verifying no packet") + logger.info ("Verifying no packet") verify_no_packet_any(self, masked_exp_pkt, self.t2_ports) # Verify ECMP: if check_ecmp: - # Check #1 : All addresses have been used. - if set(nhs) - set(returned_ip_addresses.keys()) == set([]): - print (" Each address has been used") - print ("Packets sent:{} distribution:".format(self.packet_count)) - for nh_address in returned_ip_addresses.keys(): - print (" {} : {}".format(nh_address, returned_ip_addresses[nh_address])) - # Check #2 : The packets are almost equally distributed. - # We calculate the total number of packets sent, and average it across the number of ECMP hops. - # Every next-hop should have received within 1% of the packets that we sent per nexthop. - # This check is valid only if there are large enough number of packets. - if self.packet_count > 300: - tolerance = 0.01 - for nh_address in returned_ip_addresses.keys(): - if ((1.0-tolerance) * self.packet_count > returned_ip_addresses[nh_address] - or (1.0+tolerance) * self.packet_count < returned_ip_addresses[nh_address]): - raise RuntimeError("ECMP nexthop address: {} received too less or too many of the " - "packets expected. Expected:{}, received on that address:{}".format(nh_address, self.packet_count, returned_ip_addresses[nh_address])) - - else: - raise RuntimeError('''ECMP might have failed for:{}, we expected every ip address in the nexthop group({} of them) - to be used, but only {} are used:\nUsed addresses:{}\nUnused Addresses:{}'''.format(destination, - len(nhs), len(returned_ip_addresses.keys()), - returned_ip_addresses.keys(), set(nhs)-set(returned_ip_addresses.keys()))) + self.verify_all_addresses_used_equally(nhs, returned_ip_addresses) + pkt.load = '0' * 60 + str(len(self.packets)) self.packets.append((ptf_port, str(pkt).encode("base64"))) finally: - print + logger.info("") diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index c060df6f34a..61e5c678860 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -63,8 +63,8 @@ SUPPORTED_ENCAP_TYPES = ['v4_in_v4', 'v4_in_v6', 'v6_in_v4', 'v6_in_v6'] # Starting prefixes to be used for the destinations and End points. -Destination_Prefix = 150 -NextHop_Prefix = 100 +DESTINATION_PREFIX = 150 +NEXTHOP_PREFIX = 100 pytestmark = [ # This script supports any T1 topology: t1, t1-64-lag, t1-lag. @@ -201,11 +201,11 @@ def get_ethernet_to_neighbors(neighbor_type, minigraph_data): ''' pattern = re.compile("{}$".format(neighbor_type)) - ret_list = [] + ret_list = {} for intf in minigraph_data['minigraph_neighbors']: if pattern.search(minigraph_data['minigraph_neighbors'][intf]['name']): - ret_list.append(intf) + ret_list[intf] = minigraph_data['minigraph_neighbors'][intf]['address'] return ret_list @@ -456,14 +456,44 @@ def get_t2_ports(duthost, minigraph_data): ret_list.extend([int(x[8:]) for x in list_of_ethernet_to_T2]) return ret_list -def bgp_established(duthost): +def bgp_established(duthost, ignore_list=[]): bgp_facts = duthost.bgp_facts()['ansible_facts'] for k, v in bgp_facts['bgp_neighbors'].items(): if v['state'] != 'established': - Logger.info("Neighbor %s not established yet: %s", k, v['state']) - return False + if k in ignore_list: + next + else: + Logger.info("Neighbor %s not established yet: %s", k, v['state']) + return False return True +def get_downed_bgp_neighbors(shut_intf_list, minigraph_data): + ''' + Get the list of bgp neighbors that should be down, + based on the interfaces that are shutdown. + ''' + ret_list = [] + for intf in shut_intf_list: + for m_intf in minigraph_data['minigraph_portchannel_interfaces']+minigraph_data['minigraph_interfaces']: + if m_intf['attachto'] == intf: + ret_list.append(m_intf['peer_addr']) + return ret_list + +def get_corresponding_ports(shut_intf_list, minigraph_data): + ''' + This is for tests that shutdown some of the T2 ports. + This function will check which ports are to be ignored for the encap packets coming + back to the PTF. If the encap packet comes in any of these ports, it is a bug. + ''' + return_list = [] + for intf in shut_intf_list: + if "Ethernet" in intf: + return_list.append(int(intf[8:])) + elif "PortChannel" in intf: + for port in get_ethernet_ports([intf], minigraph_data): + return_list.append(int(port[8:])) + return return_list + def get_ethernet_ports(intf_list, minigraph_data): ''' The given interface list can be either Ethernet or Portchannel. @@ -555,8 +585,8 @@ def setUp(duthosts, ptfhost, request, rand_one_dut_hostname, minigraph_facts, number_of_available_nexthops=request.config.option.total_number_of_endpoints, number_of_ecmp_nhs=request.config.option.total_number_of_nexthops, dest_af=payload_version, - dest_net_prefix=Destination_Prefix, # Hardcoded to avoid conflicts with topology networks. - nexthop_prefix=NextHop_Prefix, # Hardcoded to avoid conflicts with topology networks. + dest_net_prefix=DESTINATION_PREFIX, + nexthop_prefix=NEXTHOP_PREFIX, nh_af=outer_layer_version) data[encap_type] = encap_type_data @@ -581,14 +611,20 @@ def setUp(duthosts, ptfhost, request, rand_one_dut_hostname, minigraph_facts, encap_type = "{}_in_{}".format(payload_version, outer_layer_version) set_routes_in_dut(data['duthost'], data[encap_type]['dest_to_nh_map'], payload_version, "DEL") - for intf in data[encap_type]['vnet_intf_map'].keys(): - for entry in minigraph_facts['minigraph_interfaces'] + minigraph_facts['minigraph_portchannel_interfaces']: - if intf == entry['attachto']: - data['duthost'].shell("sudo config interface ip add {} {}".format(intf, entry['subnet'])) + for intf in data[encap_type]['selected_interfaces']: + redis_string = "INTERFACE" + if "PortChannel" in intf > 0: + redis_string = "PORTCHANNEL_INTERFACE" + data['duthost'].shell("redis-cli -n 4 hdel \"{}|{}\" vnet_name".format(redis_string, intf)) - for vnet in data[encap_type]['vnet_vni_map'].keys(): - data['duthost'].shell("redis-cli -n 4 del \"VNET|{}\"".format(vnet)) + # Our setup code re-uses same vnets for v4inv4 and v6inv4. + # There will be same vnet in multiple encap types. + # So remove vnets *after* removing the routes first. + for encap_type in SUPPORTED_ENCAP_TYPES: + for vnet in data[encap_type]['vnet_vni_map'].keys(): + data['duthost'].shell("redis-cli -n 4 del \"VNET|{}\"".format(vnet)) + time.sleep(5) for tunnel in tunnel_names.values(): data['duthost'].shell("redis-cli -n 4 del \"VXLAN_TUNNEL|{}\"".format(tunnel)) @@ -634,17 +670,17 @@ def dump_self_info_and_run_ptf(self, tcname, encap_type, expect_encap_success, p class Test_VxLAN_route_tests(Test_VxLAN): def test_vxlan_single_endpoint(self, setUp, encap_type): + ''' + tc1:Create a tunnel route to a single endpoint a. Send packets to the route prefix dst. + ''' self.setup = setUp - Logger.info("tc1:Create a tunnel route to a single endpoint a. Send packets to the route prefix dst.") self.dump_self_info_and_run_ptf("tc1", encap_type, True) def test_vxlan_modify_route_different_endpoint(self, setUp, request, encap_type): - ######################################################################################### - # testcae 2: change the route to different endpoint. - ######################################################################################### + ''' + tc2: change the route to different endpoint. packets are received only at endpoint b.") + ''' self.setup = setUp - Logger.info("tc2: change the route to different endpoint. packets are received only at endpoint b.") - # Choose a vnet vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] @@ -654,7 +690,7 @@ def test_vxlan_modify_route_different_endpoint(self, setUp, request, encap_type) # Create a new endpoint, or endpoint-list. tc2_new_end_point_list = [] for i in range(int(request.config.option.ecmp_nhs_per_destination)): - tc2_new_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + tc2_new_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) # Map the destination to the new endpoint(s). self.setup[encap_type]['dest_to_nh_map'][vnet][tc2_dest] = tc2_new_end_point_list @@ -676,7 +712,9 @@ def test_vxlan_modify_route_different_endpoint(self, setUp, request, encap_type) self.dump_self_info_and_run_ptf("tc2", encap_type, True) def test_vxlan_remove_all_route(self, setUp, encap_type): - Logger.info("tc3: remove the tunnel route. send packets to the route prefix dst. packets should not be received at any ports with dst ip of b") + ''' + tc3: remove the tunnel route. send packets to the route prefix dst. packets should not be received at any ports with dst ip of b") + ''' self.setup = setUp try: # Remove the existing routes in the DUT. @@ -689,8 +727,10 @@ def test_vxlan_remove_all_route(self, setUp, encap_type): class Test_VxLAN_ecmp_create(Test_VxLAN): def test_vxlan_configure_route1_ecmp_group_a(self, setUp, encap_type): + ''' + tc4:create tunnel route 1 with two endpoints a = {a1, a2...}. send packets to the route 1's prefix dst. packets are received at either a1 or a2. + ''' self.setup = setUp - Logger.info("tc4:create tunnel route 1 with two endpoints a = {a1, a2...}. send packets to the route 1's prefix dst. packets are received at either a1 or a2") # Choose a vnet. vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] @@ -698,10 +738,10 @@ def test_vxlan_configure_route1_ecmp_group_a(self, setUp, encap_type): # Create a new list of endpoint(s). tc4_end_point_list = [] for i in range(2): - tc4_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + tc4_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) # Create a new destination - tc4_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=Destination_Prefix) + tc4_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=DESTINATION_PREFIX) # Map the new destination and the new endpoint(s). self.setup[encap_type]['dest_to_nh_map'][vnet][tc4_new_dest] = tc4_end_point_list @@ -710,12 +750,14 @@ def test_vxlan_configure_route1_ecmp_group_a(self, setUp, encap_type): tc4_config = '[\n' + create_single_route(vnet, tc4_new_dest, HOST_MASK[get_payload_version(encap_type)], tc4_end_point_list, "SET") + '\n]' apply_config_in_swss(self.setup['duthost'], tc4_config, "vnet_route_tc4_"+encap_type) - # Verify that the new config takes effect and runs traffic. + # Verify that the new config takes effect and run traffic. self.dump_self_info_and_run_ptf("tc4", encap_type, True) def test_vxlan_configure_route1_ecmp_group_b(self, setUp, encap_type): + ''' + tc5: set tunnel route 2 to endpoint group a = {a1, a2}. send packets to route 2"s prefix dst. packets are received at either a1 or a2 + ''' self.setup = setUp - Logger.info('tc5: set tunnel route 2 to endpoint group a = {a1, a2}. send packets to route 2"s prefix dst. packets are received at either a1 or a2') self.setup_route2_ecmp_group_b(encap_type) # Verify the configs work and traffic flows correctly. self.dump_self_info_and_run_ptf("tc5", encap_type, True) @@ -730,7 +772,7 @@ def setup_route2_ecmp_group_b(self, encap_type): tc5_end_point_list = self.setup[encap_type]['dest_to_nh_map'][vnet].values()[0] # Create a new destination to use. - tc5_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=Destination_Prefix) + tc5_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=DESTINATION_PREFIX) # Map the new destination to the endpoint. self.setup[encap_type]['dest_to_nh_map'][vnet][tc5_new_dest] = tc5_end_point_list @@ -741,9 +783,11 @@ def setup_route2_ecmp_group_b(self, encap_type): self.setup[encap_type]['tc5_dest'] = tc5_new_dest def test_vxlan_configure_route2_ecmp_group_b(self, setUp, encap_type): + ''' + tc6: set tunnel route 2 to endpoint group b = {b1, b2}. send packets to route 2"s prefix dst. packets are received at either b1 or b2. + ''' self.setup = setUp self.setup_route2_ecmp_group_b(encap_type) - Logger.info('tc6: set tunnel route 2 to endpoint group b = {b1, b2}. send packets to route 2"s prefix dst. packets are received at either b1 or b2') # Choose a vnet for testing. vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] @@ -751,7 +795,7 @@ def test_vxlan_configure_route2_ecmp_group_b(self, setUp, encap_type): # Create a new list of endpoints. tc6_end_point_list = [] for i in range(2): - tc6_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + tc6_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) # Choose one of the existing destinations. tc6_new_dest = self.setup[encap_type]['tc5_dest'] @@ -777,7 +821,7 @@ def setup_route2_single_endpoint(self, encap_type): # Choose a route 2 destination and a new single endpoint for it. tc8_new_dest = self.setup[encap_type]['dest_to_nh_map'][vnet].keys()[0] - tc8_new_nh = get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix) + tc8_new_nh = get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX) Logger.info("Using destinations: dest:{} => nh:{}".format(tc8_new_dest, tc8_new_nh)) # Map the destination and new endpoint. @@ -796,14 +840,26 @@ def setup_route2_shared_endpoints(self, encap_type): # Choose a vnet for testing. vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Select 2 already existing destinations. + # Select 2 already existing destinations. They must have 2 different nexthops. tc9_new_dest1 = self.setup[encap_type]['tc8_dest'] + nh1 = self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1][0] + + nh2 = None + for dest in self.setup[encap_type]['dest_to_nh_map'][vnet].keys(): + nexthops = self.setup[encap_type]['dest_to_nh_map'][vnet][dest] + for nh in nexthops: + if nh == nh1: + next + else: + nh2 = nh + break + if nh2: + Logger.info("Using destinations: dest:{}, nexthops:{}, {}".format(tc9_new_dest1, nh1, nh2)) + else: + raise RuntimeError("Couldnot find different nexthop for this test. The current list: {}".format(self.setup[encap_type]['dest_to_nh_map'])) - tc9_new_dest2 = self.setup[encap_type]['dest_to_nh_map'][vnet].keys()[0] - Logger.info("Using destinations: dest1:{}, dest2:{}".format(tc9_new_dest1, tc9_new_dest2)) - - # Combine the first of their tunnel endpoints. - tc9_new_nhs = [self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1][0]] + [self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest2][0]] + # Use the selected nexthops(tunnel endpoints). They are guaranteed to be different. + tc9_new_nhs = [nh1, nh2] # Map the destination 1 to the combined list. self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1] = tc9_new_nhs @@ -814,8 +870,10 @@ def setup_route2_shared_endpoints(self, encap_type): self.setup[encap_type]['tc9_dest'] = tc9_new_dest1 def test_vxlan_remove_route2(self, setUp, encap_type): + ''' + tc7:send packets to route 1's prefix dst. by removing route 2 from group a, no change expected to route 1. + ''' self.setup = setUp - Logger.info("tc7:send packets to route 1's prefix dst. by removing route 2 from group a, no change expected to route 1.") # Pick a vnet for testing. vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] @@ -823,11 +881,11 @@ def test_vxlan_remove_route2(self, setUp, encap_type): # Setup: Create two destinations with the same endpoint group. tc7_end_point_list = [] for i in range(2): - tc7_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + tc7_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) tc7_destinations = [] for i in range(2): - tc7_destinations.append(get_ip_address(af=get_payload_version(encap_type), netid=Destination_Prefix)) + tc7_destinations.append(get_ip_address(af=get_payload_version(encap_type), netid=DESTINATION_PREFIX)) # Map the new destinations to the same endpoint list. for i in range(2): @@ -856,25 +914,30 @@ def test_vxlan_remove_route2(self, setUp, encap_type): self.dump_self_info_and_run_ptf("tc7", encap_type, True) def test_vxlan_route2_single_nh(self, setUp, encap_type): + ''' + tc8: set tunnel route 2 to single endpoint b1. send packets to route 2's prefix dst. + ''' self.setup = setUp - Logger.info("tc8: set tunnel route 2 to single endpoint b1. send packets to route 2's prefix dst") self.setup_route2_single_endpoint(encap_type) self.dump_self_info_and_run_ptf("tc8", encap_type, True) def test_vxlan_route2_shared_nh(self, setUp, encap_type): + ''' + tc9: set tunnel route 2 to shared endpoints a1 and b1. send packets to route 2's prefix dst. + ''' self.setup = setUp - Logger.info("tc9: set tunnel route 2 to shared endpoints a1 and b1. send packets to route 2's prefix dst") self.setup_route2_shared_endpoints(encap_type) self.dump_self_info_and_run_ptf("tc9", encap_type, True) def test_vxlan_remove_ecmp_route2(self, setUp, encap_type): + ''' + tc10: remove tunnel route 2. send packets to route 2's prefix dst. + ''' self.setup = setUp self.setup_route2_shared_endpoints(encap_type) # backup the current route config. full_map = dict(self.setup[encap_type]['dest_to_nh_map']) - Logger.info("tc10: remove tunnel route 2. send packets to route 2's prefix dst") - # This is to keep track if the selected route should be deleted in the end. del_needed = False try: @@ -919,17 +982,19 @@ def test_vxlan_remove_ecmp_route2(self, setUp, encap_type): class Test_VxLAN_ecmp_random_hash(Test_VxLAN): def test_vxlan_random_hash(self, setUp, encap_type): + ''' + tc11: set tunnel route 3 to endpoint group c = {c1, c2, c3}. ensure c1, c2, and c3 matches to underlay default route. send 1000 pkt with random hash to route 3's prefix dst. + ''' self.setup = setUp - Logger.info("tc11: set tunnel route 3 to endpoint group c = {c1, c2, c3}. ensure c1, c2, and c3 matches to underlay default route. send 1000 pkt with random hash to route 3's prefix dst") # Chose a vnet for testing. vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] # Create a new destination and 3 nhs for it. - tc11_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=Destination_Prefix) + tc11_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=DESTINATION_PREFIX) tc11_new_nhs = [] for i in range(3): - tc11_new_nhs.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NextHop_Prefix)) + tc11_new_nhs.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) # the topology always provides the default routes for any ip address. # so it is already taken care of. @@ -945,8 +1010,10 @@ def test_vxlan_random_hash(self, setUp, encap_type): class Test_VxLAN_underlay_ecmp(Test_VxLAN): @pytest.mark.parametrize("ecmp_path_count", [1, 2]) def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, ecmp_path_count): + ''' + tc12: modify the underlay default route nexthop/s. send packets to route 3's prefix dst. + ''' self.setup = setUp - Logger.info("tc12: modify the underlay default route nexthop/s. send packets to route 3's prefix dst.") # First step: pick one or two of the interfaces connected to t2, and bring them down. # verify that the encap is still working, and ptf receives the traffic. # Bring them back up. @@ -960,54 +1027,64 @@ def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, if not all_intfs: raise RuntimeError("no interface found connected to t2 neighbors. pls check the testbed, aborting.") - # Backup + # Keep a copy of the internal housekeeping list of t2 ports. + # This is the full list of DUT ports connected to T2 neighbors. + # It is one of the arguments to the ptf code. all_t2_ports = list(self.setup[encap_type]['t2_ports']) + + # A distinction in this script between ports and interfaces: + # Ports are physical (Ethernet) only. + # Interfaces have IP address(Ethernet or PortChannel). + try: - shut_intfs = [] + selected_intfs = [] # Choose some intfs based on the parameter ecmp_path_count. # when ecmp_path_count == 1, it is non-ecmp. The switching happens between ecmp and non-ecmp. # Otherwise, the switching happens within ecmp only. for i in range(ecmp_path_count): - shut_intfs.append(all_intfs[i]) + selected_intfs.append(all_intfs[i]) - for intf in shut_intfs: + for intf in selected_intfs: self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) + downed_ports = get_corresponding_ports(selected_intfs, minigraph_facts) + self.setup[encap_type]['t2_ports'] = list(set(all_t2_ports) - set(downed_ports)) + downed_bgp_neighbors = get_downed_bgp_neighbors(selected_intfs, minigraph_facts) + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost'], downed_bgp_neighbors), "BGP neighbors didn't come up after all interfaces have been brought up.") self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) - # Bring up the selected intfs. - - self.setup[encap_type]['t2_ports'] = list(all_t2_ports) - for intf in shut_intfs: + # Reverse the action: bring up the selected_intfs and shutdown others. + for intf in selected_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) - time.sleep(60) # Shutdown other interfaces. - for intf in set(all_intfs) - set(shut_intfs): + remaining_interfaces = list(set(all_intfs) - set(selected_intfs)) + for intf in remaining_interfaces: self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) - # allow time for recovery. - time.sleep(60) + downed_bgp_neighbors = get_downed_bgp_neighbors(remaining_interfaces, minigraph_facts) + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost'], downed_bgp_neighbors), "BGP neighbors didn't come up after all interfaces have been brought up.") + self.setup[encap_type]['t2_ports'] = get_corresponding_ports(selected_intfs, minigraph_facts) self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) - # Recover all interfaces. - self.setup[encap_type]['t2_ports'] = list(all_t2_ports) + # recovery. Bring all up, and verify traffic works. for intf in all_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) - # Wait for all bgp is up. - pytest_assert(wait_until(300, 30, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") - + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") # Verify traffic flows after recovery. + self.setup[encap_type]['t2_ports'] = all_t2_ports self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) finally: # If anything goes wrong in the try block, atleast bring the intf back up. - self.setup[encap_type]['t2_ports'] = list(all_t2_ports) + self.setup[encap_type]['t2_ports'] = all_t2_ports for intf in all_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) - pytest_assert(wait_until(300, 30, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") def test_vxlan_remove_add_underlay_default(self, setUp, minigraph_facts, encap_type): + ''' + tc13: remove the underlay default route. + ''' self.setup = setUp - Logger.info("tc13: remove the underlay default route.") # Find all the underlay default routes' interfaces. This means all T2 interfaces. all_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) @@ -1025,14 +1102,16 @@ def test_vxlan_remove_add_underlay_default(self, setUp, minigraph_facts, encap_t # Verify that traffic is not flowing through. self.dump_self_info_and_run_ptf("tc12", encap_type, False) - Logger.info("tc14: Re-add the underlay default route.") + ''' + tc14: Re-add the underlay default route. + ''' # Bring up the T2 interfaces. for intf in all_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) # Wait for all bgp is up. - pytest_assert(wait_until(300, 30, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") # Verify the traffic is flowing through, again. self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) @@ -1041,4 +1120,4 @@ def test_vxlan_remove_add_underlay_default(self, setUp, minigraph_facts, encap_t # If anything goes wrong in the try block, atleast bring the intf back up. for intf in all_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) - pytest_assert(wait_until(300, 30, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") From a7c803cb3dc6e00560bbc7901161bfe28190f2cd Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Tue, 29 Mar 2022 01:10:17 +0000 Subject: [PATCH 04/11] Final review comments, from Ihor Chekh, and roysr-nv --- .../test/files/ptftests/vxlan_traffic.py | 10 +- tests/vxlan/test_vxlan_ecmp.py | 237 +++++++++++------- 2 files changed, 149 insertions(+), 98 deletions(-) diff --git a/ansible/roles/test/files/ptftests/vxlan_traffic.py b/ansible/roles/test/files/ptftests/vxlan_traffic.py index 47f981650e5..46ebfd166f6 100644 --- a/ansible/roles/test/files/ptftests/vxlan_traffic.py +++ b/ansible/roles/test/files/ptftests/vxlan_traffic.py @@ -70,7 +70,7 @@ def setUp(self): # The ECMP check fails occasionally if there is not enough packets. # We should keep the packet count atleast MIN_PACKET_COUNT. if self.packet_count < MIN_PACKET_COUNT: - logger.warn("Packet_count is below minimum, resetting to {}", MIN_PACKET_COUNT) + logger.warning("Packet_count is below minimum, resetting to {}", MIN_PACKET_COUNT) self.packet_count = MIN_PACKET_COUNT self.random_mac = "00:aa:bb:cc:dd:ee" @@ -152,10 +152,10 @@ def verify_all_addresses_used_equally(self, nhs, returned_ip_addresses): ''' # Check #1 : All addresses have been used. if set(nhs) - set(returned_ip_addresses.keys()) == set([]): - logger.info (" Each address has been used") - logger.info ("Packets sent:{} distribution:".format(self.packet_count)) + logger.info(" Each address has been used") + logger.info("Packets sent:{} distribution:".format(self.packet_count)) for nh_address in returned_ip_addresses.keys(): - logger.info (" {} : {}".format(nh_address, returned_ip_addresses[nh_address])) + logger.info(" {} : {}".format(nh_address, returned_ip_addresses[nh_address])) # Check #2 : The packets are almost equally distributed. # Every next-hop should have received within 1% of the packets that we sent per nexthop(which is self.packet_count). # This check is valid only if there are large enough number of packets(300). Any lower number will need higher tolerance(more than 2%). @@ -289,7 +289,7 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, else: check_ecmp = False - logger.info ("Verifying no packet") + logger.info("Verifying no packet") verify_no_packet_any(self, masked_exp_pkt, self.t2_ports) # Verify ECMP: diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index 61e5c678860..389adbc67de 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -42,7 +42,7 @@ from tests.common.utilities import wait_until from tests.ptf_runner import ptf_runner -Logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) # Some of the Constants used in this script. Constants = {} @@ -459,12 +459,9 @@ def get_t2_ports(duthost, minigraph_data): def bgp_established(duthost, ignore_list=[]): bgp_facts = duthost.bgp_facts()['ansible_facts'] for k, v in bgp_facts['bgp_neighbors'].items(): - if v['state'] != 'established': - if k in ignore_list: - next - else: - Logger.info("Neighbor %s not established yet: %s", k, v['state']) - return False + if v['state'] != 'established' and k not in ignore_list: + logger.info("Neighbor %s not established yet: %s", k, v['state']) + return False return True def get_downed_bgp_neighbors(shut_intf_list, minigraph_data): @@ -485,13 +482,14 @@ def get_corresponding_ports(shut_intf_list, minigraph_data): This function will check which ports are to be ignored for the encap packets coming back to the PTF. If the encap packet comes in any of these ports, it is a bug. ''' - return_list = [] + eth_ifaces_list = [] for intf in shut_intf_list: if "Ethernet" in intf: - return_list.append(int(intf[8:])) + eth_ifaces_list.append(intf) elif "PortChannel" in intf: for port in get_ethernet_ports([intf], minigraph_data): - return_list.append(int(port[8:])) + eth_ifaces_list.append(port) + return_list = [minigraph_data["minigraph_ptf_indices"][iface] for iface in eth_ifaces_list] return return_list def get_ethernet_ports(intf_list, minigraph_data): @@ -526,7 +524,7 @@ def setUp(duthosts, ptfhost, request, rand_one_dut_hostname, minigraph_facts, # as DUT. Constants['DUT_HOSTID'] = request.config.option.dut_hostid - Logger.info("Constants to be used in the script:%s", Constants) + logger.info("Constants to be used in the script:%s", Constants) data = {} data['ptfhost'] = ptfhost @@ -681,21 +679,21 @@ def test_vxlan_modify_route_different_endpoint(self, setUp, request, encap_type) tc2: change the route to different endpoint. packets are received only at endpoint b.") ''' self.setup = setUp - # Choose a vnet + logger.info("Choose a vnet") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Choose a destination, which is already present. + logger.info("Choose a destination, which is already present.") tc2_dest = self.setup[encap_type]['dest_to_nh_map'][vnet].keys()[0] - # Create a new endpoint, or endpoint-list. + logger.info("Create a new endpoint, or endpoint-list.") tc2_new_end_point_list = [] for i in range(int(request.config.option.ecmp_nhs_per_destination)): tc2_new_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) - # Map the destination to the new endpoint(s). + logger.info("Map the destination to the new endpoint(s).") self.setup[encap_type]['dest_to_nh_map'][vnet][tc2_dest] = tc2_new_end_point_list - # Create the json and apply the config in the DUT swss. + logger.info("Create the json and apply the config in the DUT swss.") # The config looks like: # [ # { @@ -708,7 +706,7 @@ def test_vxlan_modify_route_different_endpoint(self, setUp, request, encap_type) tc2_full_config = '[\n' + create_single_route(vnet, tc2_dest, HOST_MASK[get_payload_version(encap_type)], tc2_new_end_point_list, "SET") + '\n]' apply_config_in_swss(self.setup['duthost'], tc2_full_config, "vnet_route_tc2_"+encap_type) - # Copy the new set of configs to the PTF and run the tests. + logger.info("Copy the new set of configs to the PTF and run the tests.") self.dump_self_info_and_run_ptf("tc2", encap_type, True) def test_vxlan_remove_all_route(self, setUp, encap_type): @@ -717,12 +715,12 @@ def test_vxlan_remove_all_route(self, setUp, encap_type): ''' self.setup = setUp try: - # Remove the existing routes in the DUT. + logger.info("Remove the existing routes in the DUT.") set_routes_in_dut(self.setup['duthost'], self.setup[encap_type]['dest_to_nh_map'], get_payload_version(encap_type), "DEL") - # Verify that the traffic is not coming back. + logger.info("Verify that the traffic is not coming back.") self.dump_self_info_and_run_ptf("tc3", encap_type, False) finally: - # Restore the routes in the DUT. + logger.info("Restore the routes in the DUT.") set_routes_in_dut(self.setup['duthost'], self.setup[encap_type]['dest_to_nh_map'], get_payload_version(encap_type), "SET") class Test_VxLAN_ecmp_create(Test_VxLAN): @@ -732,25 +730,25 @@ def test_vxlan_configure_route1_ecmp_group_a(self, setUp, encap_type): ''' self.setup = setUp - # Choose a vnet. + logger.info("Choose a vnet.") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Create a new list of endpoint(s). + logger.info("Create a new list of endpoint(s).") tc4_end_point_list = [] for i in range(2): tc4_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) - # Create a new destination + logger.info("Create a new destination") tc4_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=DESTINATION_PREFIX) - # Map the new destination and the new endpoint(s). + logger.info("Map the new destination and the new endpoint(s).") self.setup[encap_type]['dest_to_nh_map'][vnet][tc4_new_dest] = tc4_end_point_list - # Create a new config and Copy to the DUT. + logger.info("Create a new config and Copy to the DUT.") tc4_config = '[\n' + create_single_route(vnet, tc4_new_dest, HOST_MASK[get_payload_version(encap_type)], tc4_end_point_list, "SET") + '\n]' apply_config_in_swss(self.setup['duthost'], tc4_config, "vnet_route_tc4_"+encap_type) - # Verify that the new config takes effect and run traffic. + logger.info("Verify that the new config takes effect and run traffic.") self.dump_self_info_and_run_ptf("tc4", encap_type, True) def test_vxlan_configure_route1_ecmp_group_b(self, setUp, encap_type): @@ -759,25 +757,25 @@ def test_vxlan_configure_route1_ecmp_group_b(self, setUp, encap_type): ''' self.setup = setUp self.setup_route2_ecmp_group_b(encap_type) - # Verify the configs work and traffic flows correctly. + logger.info("Verify the configs work and traffic flows correctly.") self.dump_self_info_and_run_ptf("tc5", encap_type, True) def setup_route2_ecmp_group_b(self, encap_type): if self.setup[encap_type].get('tc5_dest', None): return - # Choose a vnet for testing. + logger.info("Choose a vnet for testing.") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Select an existing endpoint. + logger.info("Select an existing endpoint.") tc5_end_point_list = self.setup[encap_type]['dest_to_nh_map'][vnet].values()[0] - # Create a new destination to use. + logger.info("Create a new destination to use.") tc5_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=DESTINATION_PREFIX) - # Map the new destination to the endpoint. + logger.info("Map the new destination to the endpoint.") self.setup[encap_type]['dest_to_nh_map'][vnet][tc5_new_dest] = tc5_end_point_list - # Create the new config and apply to the DUT. + logger.info("Create the new config and apply to the DUT.") tc5_config = '[\n' + create_single_route(vnet, tc5_new_dest, HOST_MASK[get_payload_version(encap_type)], tc5_end_point_list, "SET") + '\n]' apply_config_in_swss(self.setup['duthost'], tc5_config, "vnet_route_tc5_"+encap_type) self.setup[encap_type]['tc5_dest'] = tc5_new_dest @@ -789,25 +787,25 @@ def test_vxlan_configure_route2_ecmp_group_b(self, setUp, encap_type): self.setup = setUp self.setup_route2_ecmp_group_b(encap_type) - # Choose a vnet for testing. + logger.info("Choose a vnet for testing.") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Create a new list of endpoints. + logger.info("Create a new list of endpoints.") tc6_end_point_list = [] for i in range(2): tc6_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) - # Choose one of the existing destinations. + logger.info("Choose one of the existing destinations.") tc6_new_dest = self.setup[encap_type]['tc5_dest'] - # Map the destination to the new endpoints. + logger.info("Map the destination to the new endpoints.") self.setup[encap_type]['dest_to_nh_map'][vnet][tc6_new_dest] = tc6_end_point_list - # Crete the config and apply on the DUT. + logger.info("Create the config and apply on the DUT.") tc6_config = '[\n' + create_single_route(vnet, tc6_new_dest, HOST_MASK[get_payload_version(encap_type)], tc6_end_point_list, "SET") + '\n]' apply_config_in_swss(self.setup['duthost'], tc6_config, "vnet_route_tc6_"+encap_type) - # Verify that the traffic works. + logger.info("Verify that the traffic works.") self.dump_self_info_and_run_ptf("tc6", encap_type, True) class Test_VxLAN_NHG_Modify(Test_VxLAN): @@ -816,19 +814,19 @@ def setup_route2_single_endpoint(self, encap_type): if self.setup[encap_type].get('tc8_dest', None): return - # Pick a vnet for testing. + logger.info("Pick a vnet for testing.") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Choose a route 2 destination and a new single endpoint for it. + logger.info("Choose a route 2 destination and a new single endpoint for it.") tc8_new_dest = self.setup[encap_type]['dest_to_nh_map'][vnet].keys()[0] tc8_new_nh = get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX) - Logger.info("Using destinations: dest:{} => nh:{}".format(tc8_new_dest, tc8_new_nh)) + logger.info("Using destinations: dest:{} => nh:{}".format(tc8_new_dest, tc8_new_nh)) - # Map the destination and new endpoint. + logger.info("Map the destination and new endpoint.") tc8_config = '[\n' + create_single_route(vnet, tc8_new_dest, HOST_MASK[get_payload_version(encap_type)], [tc8_new_nh], "SET") + '\n]' self.setup[encap_type]['dest_to_nh_map'][vnet][tc8_new_dest] = [tc8_new_nh] - # Apply the new config in the DUT and run traffic test. + logger.info("Apply the new config in the DUT and run traffic test.") apply_config_in_swss(self.setup['duthost'], tc8_config, "vnet_route_tc8_"+encap_type) self.setup[encap_type]['tc8_dest'] = tc8_new_dest @@ -837,10 +835,10 @@ def setup_route2_shared_endpoints(self, encap_type): return self.setup_route2_single_endpoint(encap_type) - # Choose a vnet for testing. + logger.info("Choose a vnet for testing.") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Select 2 already existing destinations. They must have 2 different nexthops. + logger.info("Select 2 already existing destinations. They must have 2 different nexthops.") tc9_new_dest1 = self.setup[encap_type]['tc8_dest'] nh1 = self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1][0] @@ -849,36 +847,79 @@ def setup_route2_shared_endpoints(self, encap_type): nexthops = self.setup[encap_type]['dest_to_nh_map'][vnet][dest] for nh in nexthops: if nh == nh1: - next + continue else: nh2 = nh break if nh2: - Logger.info("Using destinations: dest:{}, nexthops:{}, {}".format(tc9_new_dest1, nh1, nh2)) + logger.info("Using destinations: dest:{}, nexthops:{}, {}".format(tc9_new_dest1, nh1, nh2)) + else: + raise RuntimeError("Couldnot find different nexthop for this test. The current list: {}".format(self.setup[encap_type]['dest_to_nh_map'])) + + logger.info("Use the selected nexthops(tunnel endpoints). They are guaranteed to be different.") + tc9_new_nhs = [nh1, nh2] + + logger.info("Map the destination 1 to the combined list.") + self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1] = tc9_new_nhs + tc9_config = '[\n' + create_single_route(vnet, tc9_new_dest1, HOST_MASK[get_payload_version(encap_type)], tc9_new_nhs, "SET") + '\n]' + + logger.info("Apply the new config to the DUT and send traffic.") + apply_config_in_swss(self.setup['duthost'], tc9_config, "vnet_route_tc9_"+encap_type) + self.setup[encap_type]['tc9_dest'] = tc9_new_dest1 + + def setup_route2_shared_different_endpoints(self, encap_type): + if self.setup[encap_type].get('tc9_dest', None): + return + self.setup_route2_single_endpoint(encap_type) + + logger.info("Choose a vnet for testing.") + vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] + + logger.info("Select 2 already existing destinations. They must have 2 different nexthops.") + tc9_new_dest1 = self.setup[encap_type]['tc8_dest'] + old_nh = self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1][0] + + nh1 = None + nh2 = None + for dest in self.setup[encap_type]['dest_to_nh_map'][vnet].keys(): + nexthops = self.setup[encap_type]['dest_to_nh_map'][vnet][dest] + for nh in nexthops: + if nh == old_nh: + next + else: + if not nh1: + nh1 = nh + elif not nh2: + if nh != nh1: + nh2 = nh + break + if nh2: + logger.info("Using destinations: dest:{}, nexthops:{}, {}".format(tc9_new_dest1, nh1, nh2)) else: raise RuntimeError("Couldnot find different nexthop for this test. The current list: {}".format(self.setup[encap_type]['dest_to_nh_map'])) - # Use the selected nexthops(tunnel endpoints). They are guaranteed to be different. + logger.info("Use the selected nexthops(tunnel endpoints). They are guaranteed to be different.") tc9_new_nhs = [nh1, nh2] - # Map the destination 1 to the combined list. + logger.info("Map the destination 1 to the combined list.") self.setup[encap_type]['dest_to_nh_map'][vnet][tc9_new_dest1] = tc9_new_nhs tc9_config = '[\n' + create_single_route(vnet, tc9_new_dest1, HOST_MASK[get_payload_version(encap_type)], tc9_new_nhs, "SET") + '\n]' - # Apply the new config to the DUT and send traffic. + logger.info("Apply the new config to the DUT and send traffic.") apply_config_in_swss(self.setup['duthost'], tc9_config, "vnet_route_tc9_"+encap_type) self.setup[encap_type]['tc9_dest'] = tc9_new_dest1 + def test_vxlan_remove_route2(self, setUp, encap_type): ''' tc7:send packets to route 1's prefix dst. by removing route 2 from group a, no change expected to route 1. ''' self.setup = setUp - # Pick a vnet for testing. + logger.info("Pick a vnet for testing.") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Setup: Create two destinations with the same endpoint group. + logger.info("Setup: Create two destinations with the same endpoint group.") tc7_end_point_list = [] for i in range(2): tc7_end_point_list.append(get_ip_address(af=get_outer_layer_version(encap_type), netid=NEXTHOP_PREFIX)) @@ -887,30 +928,30 @@ def test_vxlan_remove_route2(self, setUp, encap_type): for i in range(2): tc7_destinations.append(get_ip_address(af=get_payload_version(encap_type), netid=DESTINATION_PREFIX)) - # Map the new destinations to the same endpoint list. + logger.info("Map the new destinations to the same endpoint list.") for i in range(2): self.setup[encap_type]['dest_to_nh_map'][vnet][tc7_destinations[i]] = tc7_end_point_list - # Apply the setup configs to the DUT. + logger.info("Apply the setup configs to the DUT.") for i in range(2): tc7_setup_config = '[\n' + create_single_route(vnet, tc7_destinations[i], HOST_MASK[get_payload_version(encap_type)], tc7_end_point_list, "SET") + '\n]' apply_config_in_swss(self.setup['duthost'], tc7_setup_config, "vnet_route_tc7_"+encap_type) - # verify the setup works. + logger.info("Verify the setup works.") self.dump_self_info_and_run_ptf("tc7", encap_type, True) - # End of setup. + logger.info("End of setup.") - # now remove one of the routes. - # Pick one out of the two TC7 destinations. + logger.info("Remove one of the routes.") + logger.info("Pick one out of the two TC7 destinations.") tc7_removed_dest = tc7_destinations[0] tc7_removed_endpoint = self.setup[encap_type]['dest_to_nh_map'][vnet][tc7_removed_dest] del self.setup[encap_type]['dest_to_nh_map'][vnet][tc7_removed_dest] - # Remove the chosen dest/endpoint from the DUT. + logger.info("Remove the chosen dest/endpoint from the DUT.") tc7_config = '[\n' + create_single_route(vnet, tc7_removed_dest, HOST_MASK[get_payload_version(encap_type)], tc7_removed_endpoint, "DEL") + '\n]' apply_config_in_swss(self.setup['duthost'], tc7_config, "vnet_route_tc7_"+encap_type) - # Verify the rest of the traffic still works. + logger.info("Verify the rest of the traffic still works.") self.dump_self_info_and_run_ptf("tc7", encap_type, True) def test_vxlan_route2_single_nh(self, setUp, encap_type): @@ -929,53 +970,61 @@ def test_vxlan_route2_shared_nh(self, setUp, encap_type): self.setup_route2_shared_endpoints(encap_type) self.dump_self_info_and_run_ptf("tc9", encap_type, True) + def test_vxlan_route2_shared_different_nh(self, setUp, encap_type): + ''' + tc9.2: set tunnel route 2 to 2 completely different shared(no-reuse) endpoints a1 and b1. send packets to route 2's prefix dst. + ''' + self.setup = setUp + self.setup_route2_shared_different_endpoints(encap_type) + self.dump_self_info_and_run_ptf("tc9", encap_type, True) + def test_vxlan_remove_ecmp_route2(self, setUp, encap_type): ''' tc10: remove tunnel route 2. send packets to route 2's prefix dst. ''' self.setup = setUp self.setup_route2_shared_endpoints(encap_type) - # backup the current route config. + logger.info("Backup the current route config.") full_map = dict(self.setup[encap_type]['dest_to_nh_map']) - # This is to keep track if the selected route should be deleted in the end. + logger.info("This is to keep track if the selected route should be deleted in the end.") del_needed = False try: - # Choose a vnet for testing. + logger.info("Choose a vnet for testing.") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Choose a destination and its nhs to delete. + logger.info("Choose a destination and its nhs to delete.") tc10_dest = self.setup[encap_type]['tc9_dest'] tc10_nhs = self.setup[encap_type]['dest_to_nh_map'][vnet][tc10_dest] - Logger.info("Using destination: dest:{}, nh:{}".format(tc10_dest, tc10_nhs)) + logger.info("Using destination: dest:{}, nh:{}".format(tc10_dest, tc10_nhs)) - # Delete the dest and nh in the DUT. + logger.info("Delete the dest and nh in the DUT.") tc10_config = '[\n' + create_single_route(vnet, tc10_dest, HOST_MASK[get_payload_version(encap_type)], tc10_nhs, "DEL") + '\n]' apply_config_in_swss(self.setup['duthost'], tc10_config, "vnet_route_tc10_"+encap_type) del_needed = True - # We should pass only the deleted entry to the ptf call, and expect encap to fail. - # Clear out the mappings, and keep only the deleted dest and nhs. + logger.info("We should pass only the deleted entry to the ptf call, and expect encap to fail.") + logger.info("Clear out the mappings, and keep only the deleted dest and nhs.") self.setup[encap_type]['dest_to_nh_map'][vnet] = {} self.setup[encap_type]['dest_to_nh_map'][vnet][tc10_dest] = tc10_nhs - # the deleted route should fail to receive traffic. + logger.info("The deleted route should fail to receive traffic.") self.dump_self_info_and_run_ptf("tc10", encap_type, False) # all others should be working. # Housekeeping: - # Restore the mapping of dest->nhs. + logger.info("Restore the mapping of dest->nhs.") self.setup[encap_type]['dest_to_nh_map'] = dict(full_map) - # Remove the deleted entry alone. + logger.info("Remove the deleted entry alone.") del self.setup[encap_type]['dest_to_nh_map'][vnet][tc10_dest] del_needed = False - # Check the traffic is working in the other routes. + logger.info("Check the traffic is working in the other routes.") self.dump_self_info_and_run_ptf("tc10", encap_type, True) except: self.setup[encap_type]['dest_to_nh_map'] = dict(full_map) - # Remove the deleted entry alone. + logger.info("Remove the deleted entry alone.") if del_needed: del self.setup[encap_type]['dest_to_nh_map'][vnet][tc10_dest] raise @@ -987,10 +1036,10 @@ def test_vxlan_random_hash(self, setUp, encap_type): ''' self.setup = setUp - # Chose a vnet for testing. + logger.info("Chose a vnet for testing.") vnet = self.setup[encap_type]['vnet_vni_map'].keys()[0] - # Create a new destination and 3 nhs for it. + logger.info("Create a new destination and 3 nhs for it.") tc11_new_dest = get_ip_address(af=get_payload_version(encap_type), netid=DESTINATION_PREFIX) tc11_new_nhs = [] for i in range(3): @@ -999,11 +1048,11 @@ def test_vxlan_random_hash(self, setUp, encap_type): # the topology always provides the default routes for any ip address. # so it is already taken care of. - # Map the new dest and nhs. + logger.info("Map the new dest and nhs.") tc11_config = '[\n' + create_single_route(vnet, tc11_new_dest, HOST_MASK[get_payload_version(encap_type)], tc11_new_nhs, "SET") + '\n]' self.setup[encap_type]['dest_to_nh_map'][vnet][tc11_new_dest] = tc11_new_nhs - # Apply the config in the DUT and verify traffic. The random hash and ECMP check is already taken care of in the VxLAN PTF script. + logger.info("Apply the config in the DUT and verify traffic. The random hash and ECMP check is already taken care of in the VxLAN PTF script.") apply_config_in_swss(self.setup['duthost'], tc11_config, "vnet_route_tc11_"+encap_type) self.dump_self_info_and_run_ptf("tc11", encap_type, True, packet_count=1000) @@ -1023,7 +1072,7 @@ def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, all_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) if not all_intfs: all_intfs = get_ethernet_to_neighbors("T2", minigraph_facts) - Logger.info("Dumping T2 link info: {}".format(all_intfs)) + logger.info("Dumping T2 link info: {}".format(all_intfs)) if not all_intfs: raise RuntimeError("no interface found connected to t2 neighbors. pls check the testbed, aborting.") @@ -1052,10 +1101,10 @@ def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost'], downed_bgp_neighbors), "BGP neighbors didn't come up after all interfaces have been brought up.") self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) - # Reverse the action: bring up the selected_intfs and shutdown others. + logger.info("Reverse the action: bring up the selected_intfs and shutdown others.") for intf in selected_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) - # Shutdown other interfaces. + logger.info("Shutdown other interfaces.") remaining_interfaces = list(set(all_intfs) - set(selected_intfs)) for intf in remaining_interfaces: self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) @@ -1064,21 +1113,22 @@ def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, self.setup[encap_type]['t2_ports'] = get_corresponding_ports(selected_intfs, minigraph_facts) self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) - # recovery. Bring all up, and verify traffic works. + logger.info("Recovery. Bring all up, and verify traffic works.") for intf in all_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) - # Wait for all bgp is up. + logger.info("Wait for all bgp is up.") pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") - # Verify traffic flows after recovery. + logger.info("Verify traffic flows after recovery.") self.setup[encap_type]['t2_ports'] = all_t2_ports self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) - finally: + except Exception: # If anything goes wrong in the try block, atleast bring the intf back up. self.setup[encap_type]['t2_ports'] = all_t2_ports for intf in all_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") + raise def test_vxlan_remove_add_underlay_default(self, setUp, minigraph_facts, encap_type): ''' @@ -1086,38 +1136,39 @@ def test_vxlan_remove_add_underlay_default(self, setUp, minigraph_facts, encap_t ''' self.setup = setUp - # Find all the underlay default routes' interfaces. This means all T2 interfaces. + logger.info("Find all the underlay default routes' interfaces. This means all T2 interfaces.") all_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) if not all_intfs: all_intfs = get_ethernet_to_neighbors("T2", minigraph_facts) - Logger.info("Dumping T2 link info: {}".format(all_intfs)) + logger.info("Dumping T2 link info: {}".format(all_intfs)) if not all_intfs: raise RuntimeError("no interface found connected to t2 neighbors. pls check the testbed, aborting.") try: - # Bring down the T2 interfaces. + logger.info("Bring down the T2 interfaces.") for intf in all_intfs: self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) - # Verify that traffic is not flowing through. + logger.info("Verify that traffic is not flowing through.") self.dump_self_info_and_run_ptf("tc12", encap_type, False) ''' tc14: Re-add the underlay default route. ''' - # Bring up the T2 interfaces. + logger.info("Bring up the T2 interfaces.") for intf in all_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) - # Wait for all bgp is up. + logger.info("Wait for all bgp is up.") pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") - # Verify the traffic is flowing through, again. + logger.info("Verify the traffic is flowing through, again.") self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) - finally: - # If anything goes wrong in the try block, atleast bring the intf back up. + except Exception: + logger.info("If anything goes wrong in the try block, atleast bring the intf back up.") for intf in all_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") + raise From 2c4d2f1a9035d2cf7619b36966520ca420212e91 Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Mon, 11 Apr 2022 04:03:27 +0000 Subject: [PATCH 05/11] Adding calculated sleep after set or del routes, as per comment from dgsudharsan --- ansible/roles/test/files/ptftests/vxlan_traffic.py | 3 +-- tests/vxlan/test_vxlan_ecmp.py | 13 +++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/roles/test/files/ptftests/vxlan_traffic.py b/ansible/roles/test/files/ptftests/vxlan_traffic.py index 8b4d50121d5..9b5ec851216 100644 --- a/ansible/roles/test/files/ptftests/vxlan_traffic.py +++ b/ansible/roles/test/files/ptftests/vxlan_traffic.py @@ -215,7 +215,6 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, pkt_opts.update(options_v6) pkt = simple_tcpv6_packet(**pkt_opts) pkt_opts['ipv6_hlim'] = 63 - pkt_opts['eth_dst'] = self.dut_mac pkt_opts['eth_src'] = self.dut_mac exp_pkt = simple_tcpv6_packet(**pkt_opts) else: @@ -288,7 +287,7 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, # Verify ECMP: if check_ecmp: self.verify_all_addresses_used_equally(nhs, returned_ip_addresses) - + pkt.load = '0' * 60 + str(len(self.packets)) self.packets.append((ptf_port, str(pkt).encode("base64"))) diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index a2879e7a14f..4314ac439ca 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -68,7 +68,7 @@ pytestmark = [ # This script supports any T1 topology: t1, t1-64-lag, t1-lag. - pytest.mark.topology("t1", "t1-64-lag", "t1-lag"), + pytest.mark.topology("t1"), pytest.mark.sanity_check(post_check=True) ] @@ -346,9 +346,9 @@ def apply_config_in_swss(duthost, config, name="swss_"): duthost.copy(content=config, dest="/tmp/{}".format(filename)) duthost.shell('docker exec -i swss swssconfig /dev/stdin < /tmp/{}'.format(filename)) + time.sleep(int(0.0005*getsizeof(config)) + 1) if not Constants['KEEP_TEMP_FILES']: duthost.shell("rm /tmp/{}".format(filename)) - time.sleep(1) def get_list_of_nexthops(number, af, prefix=100): nexthop_list = [] @@ -462,6 +462,8 @@ def bgp_established(duthost, ignore_list=[]): if v['state'] != 'established' and k not in ignore_list: logger.info("Neighbor %s not established yet: %s", k, v['state']) return False + # Now wait for the routes to be updated. + time.sleep(10) return True def get_downed_bgp_neighbors(shut_intf_list, minigraph_data): @@ -583,8 +585,8 @@ def setUp(duthosts, ptfhost, request, rand_one_dut_hostname, minigraph_facts, number_of_available_nexthops=request.config.option.total_number_of_endpoints, number_of_ecmp_nhs=request.config.option.total_number_of_nexthops, dest_af=payload_version, - dest_net_prefix=DESTINATION_PREFIX, - nexthop_prefix=NEXTHOP_PREFIX, + dest_net_prefix=DESTINATION_PREFIX, + nexthop_prefix=NEXTHOP_PREFIX, nh_af=outer_layer_version) data[encap_type] = encap_type_data @@ -615,7 +617,7 @@ def setUp(duthosts, ptfhost, request, rand_one_dut_hostname, minigraph_facts, redis_string = "PORTCHANNEL_INTERFACE" data['duthost'].shell("redis-cli -n 4 hdel \"{}|{}\" vnet_name".format(redis_string, intf)) - # Our setup code re-uses same vnets for v4inv4 and v6inv4. + # This script's setup code re-uses same vnets for v4inv4 and v6inv4. # There will be same vnet in multiple encap types. # So remove vnets *after* removing the routes first. for encap_type in SUPPORTED_ENCAP_TYPES: @@ -648,7 +650,6 @@ def dump_self_info_and_run_ptf(self, tcname, encap_type, expect_encap_success, p }, indent=4), dest=config_filename) - time.sleep(int(0.00005*getsizeof(self.setup[encap_type]['dest_to_nh_map'])) + 1) ptf_runner(self.setup['ptfhost'], "ptftests", "vxlan_traffic.VXLAN", From a9f0aef83c60a51241418b8982709568399b0cd0 Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Tue, 12 Apr 2022 02:40:08 +0000 Subject: [PATCH 06/11] Adding back the supported topologies. --- tests/vxlan/test_vxlan_ecmp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index 4314ac439ca..33fa06257df 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -68,7 +68,7 @@ pytestmark = [ # This script supports any T1 topology: t1, t1-64-lag, t1-lag. - pytest.mark.topology("t1"), + pytest.mark.topology("t1", "t1-64-lag", "t1-lag"), pytest.mark.sanity_check(post_check=True) ] From 59ae8e1e8177f4c3768933b5203e211468baa5fb Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Sat, 16 Apr 2022 07:39:09 +0000 Subject: [PATCH 07/11] Update the conditional skip criteria and revert the changes to get_ethernet_interface function --- .../plugins/conditional_mark/tests_mark_conditions.yaml | 6 +++--- tests/vxlan/test_vxlan_ecmp.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index 0ce5943a558..14c3235941b 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -343,12 +343,12 @@ system_health/test_system_health.py::test_service_checker_with_process_exit: ####################################### vxlan/test_vxlan_ecmp.py: skip: - reason: "VxLAN ECMP test is not yet supported on multi-ASIC platform" + reason: "VxLAN ECMP test is not yet supported on multi-ASIC platform. And this test can only run on 4600c and 8102" conditions: - "is_multi_asic==True" -vxlan/test_vxlan_ecmp.py::Test_VxLAN_route_tests::test_vxlan_single_endpoint: +vxlan/test_vxlan_ecmp.py: skip: - reason: "This test can only run on 4600c and 8102" + reason: "VxLAN ECMP test is supported only on 4600c and 8102" conditions: - "platform not in ['x86_64-mlnx_msn4600c-r0', 'x86_64-8102_64h_o-r0']" diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index 33fa06257df..5520d58f454 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -201,11 +201,11 @@ def get_ethernet_to_neighbors(neighbor_type, minigraph_data): ''' pattern = re.compile("{}$".format(neighbor_type)) - ret_list = {} + ret_list = [] for intf in minigraph_data['minigraph_neighbors']: if pattern.search(minigraph_data['minigraph_neighbors'][intf]['name']): - ret_list[intf] = minigraph_data['minigraph_neighbors'][intf]['address'] + ret_list.append(intf) return ret_list From af025eca0a87f4dc61267046a06de2303031f2e2 Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Sat, 16 Apr 2022 07:57:14 +0000 Subject: [PATCH 08/11] Update the skip conditional. --- .../plugins/conditional_mark/tests_mark_conditions.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index 14c3235941b..79fdfaa4939 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -343,12 +343,7 @@ system_health/test_system_health.py::test_service_checker_with_process_exit: ####################################### vxlan/test_vxlan_ecmp.py: skip: - reason: "VxLAN ECMP test is not yet supported on multi-ASIC platform. And this test can only run on 4600c and 8102" + reason: "VxLAN ECMP test is not yet supported on multi-ASIC platform. Also this test can only run on 4600c and 8102." conditions: - - "is_multi_asic==True" + - "(is_multi_asic==True) or (platform not in ['x86_64-mlnx_msn4600c-r0', 'x86_64-8102_64h_o-r0'])" -vxlan/test_vxlan_ecmp.py: - skip: - reason: "VxLAN ECMP test is supported only on 4600c and 8102" - conditions: - - "platform not in ['x86_64-mlnx_msn4600c-r0', 'x86_64-8102_64h_o-r0']" From 94325783b36c3bdb70dc668d7354d66f860b1ffd Mon Sep 17 00:00:00 2001 From: rraghav-cisco Date: Sat, 23 Apr 2022 00:40:27 +0000 Subject: [PATCH 09/11] Restrict selected_interfaces to T0 facing only. --- tests/vxlan/test_vxlan_ecmp.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index 5520d58f454..ad33ee25e28 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -131,9 +131,9 @@ def select_required_interfaces(duthost, number_of_required_interfaces, minigraph ''' Pick the required number of interfaces to use for tests. These interfaces will be selected based on if they are currently running a established BGP. - The interfaces will be picked from the T1 facing side. + The interfaces will be picked from the T0 facing side. ''' - bgp_interfaces = get_all_interfaces_running_bgp(duthost, minigraph_data) + bgp_interfaces = get_all_interfaces_running_bgp(duthost, minigraph_data, "T0") interface_ip_table = minigraph_data['minigraph_interfaces'] if interface_ip_table: available_interfaces = interface_ip_table @@ -216,16 +216,17 @@ def assign_intf_ip_address(selected_interfaces, af): intf_ip_map[intf] = ip return intf_ip_map -def get_all_interfaces_running_bgp(duthost, minigraph_data): +def get_all_interfaces_running_bgp(duthost, minigraph_data, neighbor_type): bgp_neigh_list = duthost.bgp_facts()['ansible_facts']['bgp_neighbors'] minigraph_ip_interfaces = minigraph_data['minigraph_interfaces'] + minigraph_data['minigraph_portchannel_interfaces'] peer_addr_map = {} + pattern = re.compile("{}$".format(neighbor_type)) for x in minigraph_ip_interfaces: peer_addr_map[x['peer_addr']] = {x['attachto'] : x['addr']} ret_list = {} for x, entry in peer_addr_map.iteritems(): - if bgp_neigh_list[x]['state'] == 'established': + if bgp_neigh_list[x]['state'] == 'established' and pattern.search(bgp_neigh_list[x]['description']): ret_list[x] = entry return ret_list @@ -613,7 +614,7 @@ def setUp(duthosts, ptfhost, request, rand_one_dut_hostname, minigraph_facts, for intf in data[encap_type]['selected_interfaces']: redis_string = "INTERFACE" - if "PortChannel" in intf > 0: + if "PortChannel" in intf: redis_string = "PORTCHANNEL_INTERFACE" data['duthost'].shell("redis-cli -n 4 hdel \"{}|{}\" vnet_name".format(redis_string, intf)) From 9fb0a5c1dbea6b71df09b3967138f34209481513 Mon Sep 17 00:00:00 2001 From: rraghav-cisco <58446052+rraghav-cisco@users.noreply.github.com> Date: Sat, 23 Apr 2022 00:37:52 -0700 Subject: [PATCH 10/11] Updating the traffic/ptf script to turn on ECN Turned on the ECN testing, which was turned off till now. Needed a minor modification to the ECN calculation. --- ansible/roles/test/files/ptftests/vxlan_traffic.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ansible/roles/test/files/ptftests/vxlan_traffic.py b/ansible/roles/test/files/ptftests/vxlan_traffic.py index 9b5ec851216..4392484dd2f 100644 --- a/ansible/roles/test/files/ptftests/vxlan_traffic.py +++ b/ansible/roles/test/files/ptftests/vxlan_traffic.py @@ -38,7 +38,7 @@ # Some constants used in this code MIN_PACKET_COUNT = 4 MINIMUM_PACKETS_FOR_ECMP_VALIDATION = 300 -TEST_ECN = False +TEST_ECN = True def get_incremental_value(key): global VARS @@ -174,8 +174,9 @@ def test_encap(self, ptf_port, vni, ptf_addr, destination, nhs, test_ecn=False, options = {'ip_ecn' : 0} options_v6 = {'ipv6_ecn' : 0} if test_ecn: - options = {'ip_ecn' : random.randint(0, 3)} - options_v6 = {'ipv6_ecn' : random.randint(0, 3)} + ecn = random.randint(0, 3) + options = {'ip_ecn' : ecn} + options_v6 = {'ipv6_ecn' : ecn} # ECMP support, assume it is a string of comma seperated list of addresses. returned_ip_addresses = {} From 411bd24ff1ed857b06e43dab3b232d5c8e4a05fc Mon Sep 17 00:00:00 2001 From: rraghav-cisco <58446052+rraghav-cisco@users.noreply.github.com> Date: Sat, 30 Apr 2022 16:42:05 -0700 Subject: [PATCH 11/11] Fixing bgp_established. 1. Added more stringent checks for bgp_established() function. Now it verifies the down neighbors as well as the established neighbors. 2. Fixed minor typos in TC names, comments and logs. 3. Called the bgp_established() after bringing down the interfaces in tc12. (as per review comments). --- tests/vxlan/test_vxlan_ecmp.py | 74 +++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/tests/vxlan/test_vxlan_ecmp.py b/tests/vxlan/test_vxlan_ecmp.py index ad33ee25e28..41846bee34c 100644 --- a/tests/vxlan/test_vxlan_ecmp.py +++ b/tests/vxlan/test_vxlan_ecmp.py @@ -457,12 +457,26 @@ def get_t2_ports(duthost, minigraph_data): ret_list.append(minigraph_data["minigraph_ptf_indices"][iface]) return ret_list -def bgp_established(duthost, ignore_list=[]): +def bgp_established(duthost, down_list=[]): bgp_facts = duthost.bgp_facts()['ansible_facts'] for k, v in bgp_facts['bgp_neighbors'].items(): - if v['state'] != 'established' and k not in ignore_list: - logger.info("Neighbor %s not established yet: %s", k, v['state']) - return False + if v['state'] == 'established': + if k in down_list: + # The neighbor is supposed to be down, and is actually up. + logger.info("Neighbor %s is established, but should be down.", k) + return False + else: + # The neighbor is supposed to be up, and is actually up. + continue + else: + if k in down_list: + # The neighbor is supposed to be down, and is actually down. + continue + else: + # The neighbor is supposed to be up, but is actually down. + logger.info("Neighbor %s is not yet established, has state: %s", k, v['state']) + return False + # Now wait for the routes to be updated. time.sleep(10) return True @@ -978,7 +992,7 @@ def test_vxlan_route2_shared_different_nh(self, setUp, encap_type): ''' self.setup = setUp self.setup_route2_shared_different_endpoints(encap_type) - self.dump_self_info_and_run_ptf("tc9", encap_type, True) + self.dump_self_info_and_run_ptf("tc9.2", encap_type, True) def test_vxlan_remove_ecmp_route2(self, setUp, encap_type): ''' @@ -1071,12 +1085,12 @@ def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, # After that, bring down all the other t2 interfaces, other than the ones used in the first step. # This will force a modification to the underlay default routes nexthops. - all_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) - if not all_intfs: - all_intfs = get_ethernet_to_neighbors("T2", minigraph_facts) - logger.info("Dumping T2 link info: {}".format(all_intfs)) - if not all_intfs: - raise RuntimeError("no interface found connected to t2 neighbors. pls check the testbed, aborting.") + all_t2_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) + if not all_t2_intfs: + all_t2_intfs = get_ethernet_to_neighbors("T2", minigraph_facts) + logger.info("Dumping T2 link info: {}".format(all_t2_intfs)) + if not all_t2_intfs: + raise RuntimeError("No interface found connected to t2 neighbors. pls check the testbed, aborting.") # Keep a copy of the internal housekeeping list of t2 ports. # This is the full list of DUT ports connected to T2 neighbors. @@ -1093,30 +1107,30 @@ def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, # when ecmp_path_count == 1, it is non-ecmp. The switching happens between ecmp and non-ecmp. # Otherwise, the switching happens within ecmp only. for i in range(ecmp_path_count): - selected_intfs.append(all_intfs[i]) + selected_intfs.append(all_t2_intfs[i]) for intf in selected_intfs: self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) downed_ports = get_corresponding_ports(selected_intfs, minigraph_facts) self.setup[encap_type]['t2_ports'] = list(set(all_t2_ports) - set(downed_ports)) downed_bgp_neighbors = get_downed_bgp_neighbors(selected_intfs, minigraph_facts) - pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost'], downed_bgp_neighbors), "BGP neighbors didn't come up after all interfaces have been brought up.") + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost'], down_list=downed_bgp_neighbors), "BGP neighbors didn't come up after all interfaces have been brought up.") self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) logger.info("Reverse the action: bring up the selected_intfs and shutdown others.") for intf in selected_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) logger.info("Shutdown other interfaces.") - remaining_interfaces = list(set(all_intfs) - set(selected_intfs)) + remaining_interfaces = list(set(all_t2_intfs) - set(selected_intfs)) for intf in remaining_interfaces: self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) downed_bgp_neighbors = get_downed_bgp_neighbors(remaining_interfaces, minigraph_facts) - pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost'], downed_bgp_neighbors), "BGP neighbors didn't come up after all interfaces have been brought up.") + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost'], down_list=downed_bgp_neighbors), "BGP neighbors didn't come up after all interfaces have been brought up.") self.setup[encap_type]['t2_ports'] = get_corresponding_ports(selected_intfs, minigraph_facts) self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) logger.info("Recovery. Bring all up, and verify traffic works.") - for intf in all_intfs: + for intf in all_t2_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) logger.info("Wait for all bgp is up.") pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") @@ -1127,7 +1141,7 @@ def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, except Exception: # If anything goes wrong in the try block, atleast bring the intf back up. self.setup[encap_type]['t2_ports'] = all_t2_ports - for intf in all_intfs: + for intf in all_t2_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") raise @@ -1135,42 +1149,46 @@ def test_vxlan_modify_underlay_default(self, setUp, minigraph_facts, encap_type, def test_vxlan_remove_add_underlay_default(self, setUp, minigraph_facts, encap_type): ''' tc13: remove the underlay default route. + tc14: add the underlay default route. ''' self.setup = setUp logger.info("Find all the underlay default routes' interfaces. This means all T2 interfaces.") - all_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) - if not all_intfs: - all_intfs = get_ethernet_to_neighbors("T2", minigraph_facts) - logger.info("Dumping T2 link info: {}".format(all_intfs)) - if not all_intfs: - raise RuntimeError("no interface found connected to t2 neighbors. pls check the testbed, aborting.") + all_t2_intfs = list(get_portchannels_to_neighbors(self.setup['duthost'], "T2", minigraph_facts)) + if not all_t2_intfs: + all_t2_intfs = get_ethernet_to_neighbors("T2", minigraph_facts) + logger.info("Dumping T2 link info: {}".format(all_t2_intfs)) + if not all_t2_intfs: + raise RuntimeError("No interface found connected to t2 neighbors. pls check the testbed, aborting.") try: logger.info("Bring down the T2 interfaces.") - for intf in all_intfs: + for intf in all_t2_intfs: self.setup['duthost'].shell("sudo config interface shutdown {}".format(intf)) + downed_bgp_neighbors = get_downed_bgp_neighbors(all_t2_intfs, minigraph_facts) + pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost'], down_list=downed_bgp_neighbors), + "BGP neighbors have not reached the required state after T2 intf are shutdown.") logger.info("Verify that traffic is not flowing through.") - self.dump_self_info_and_run_ptf("tc12", encap_type, False) + self.dump_self_info_and_run_ptf("tc13", encap_type, False) ''' tc14: Re-add the underlay default route. ''' logger.info("Bring up the T2 interfaces.") - for intf in all_intfs: + for intf in all_t2_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) logger.info("Wait for all bgp is up.") pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") logger.info("Verify the traffic is flowing through, again.") - self.dump_self_info_and_run_ptf("tc12", encap_type, True, packet_count=1000) + self.dump_self_info_and_run_ptf("tc14", encap_type, True, packet_count=1000) except Exception: logger.info("If anything goes wrong in the try block, atleast bring the intf back up.") - for intf in all_intfs: + for intf in all_t2_intfs: self.setup['duthost'].shell("sudo config interface startup {}".format(intf)) pytest_assert(wait_until(300, 30, 0, bgp_established, self.setup['duthost']), "BGP neighbors didn't come up after all interfaces have been brought up.") raise