diff --git a/ansible/library/announce_routes.py b/ansible/library/announce_routes.py index 19f94c87f7..4b78de4f84 100644 --- a/ansible/library/announce_routes.py +++ b/ansible/library/announce_routes.py @@ -75,6 +75,10 @@ IPV6_ADDRESS_PATTERN_DEFAULT_VALUE = '20%02X:%02X%02X:0:%02X::/64' ENABLE_IPV4_ROUTES_GENERATION_DEFAULT_VALUE = True ENABLE_IPV6_ROUTES_GENERATION_DEFAULT_VALUE = True +BGP_SCALE_T1S = [ + 't1-isolated-d254u2', 't1-isolated-d254u2s1', 't1-isolated-d254u2s2', + 't1-isolated-d510u2', 't1-isolated-d510u2s2' +] # Describe default number of COLOs COLO_NUMBER = 30 @@ -100,6 +104,8 @@ M0_SUBNET_PREFIX_LEN_V6 = 64 # Describe default start asn of M1s M1_ASN_START = 65200 +# Describe default leaf number +LEAF_NUMBER = 256 def wait_for_http(host_ip, http_port, timeout=10): @@ -461,6 +467,30 @@ def generate_routes(family, podset_number, tor_number, tor_subnet_number, return routes, suffix +def generate_t1_to_t0_routes(family, offset, leaf_number, subnet_size, tor_asn, leaf_asn_start, nexthop, nexthop_v6, + podset_num=1, ipv6_address_pattern=IPV6_ADDRESS_PATTERN_DEFAULT_VALUE): + routes = [] + for podset in range(0, podset_num): + for leaf in range(0, leaf_number): + suffix = offset + leaf + octet2 = (168 + int(suffix / (256 ** 2))) + octet1 = (192 + int(octet2 / 256)) + octet2 = (octet2 % 256) + octet3 = (int(suffix / 256) % 256) + octet4 = (suffix % 256) + prefixlen_v4 = (32 - int(math.log(subnet_size, 2))) + prefix = "{}.{}.{}.{}/{}".format(octet1, octet2, octet3, octet4, prefixlen_v4) + prefix_v6 = ipv6_address_pattern % ( + octet1, octet2, octet3, octet4) + leaf_asn = leaf_asn_start + podset + aspath = "{} {}".format(leaf_asn, tor_asn) + if family in ["v4", "both"]: + routes.append((prefix, nexthop, aspath)) + if family in ["v6", "both"]: + routes.append((prefix_v6, nexthop_v6, aspath)) + return routes, suffix + + def fib_t0(topo, ptf_ip, no_default_route=False, action="announce", upstream_neighbor_groups=0): common_config = topo['configuration_properties'].get('common', {}) podset_number = common_config.get("podset_number", PODSET_NUMBER) @@ -527,7 +557,8 @@ def fib_t0(topo, ptf_ip, no_default_route=False, action="announce", upstream_nei current_routes_offset += last_suffix -def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_default_route=False): +def fib_t1_lag(topo, ptf_ip, topo_name, no_default_route=False, action="announce", tor_default_route=False, + downstream_neighbor_groups=0): common_config = topo['configuration_properties'].get('common', {}) podset_number = common_config.get("podset_number", PODSET_NUMBER) tor_number = common_config.get("tor_number", TOR_NUMBER) @@ -553,13 +584,22 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_defa if 'DPUs' in topo['topology']: dpus = topo['topology']['DPUs'] + last_suffix = 0 + if topo_name in BGP_SCALE_T1S: + tor_default_route = True + routes_to_change = {} for k, v in vms_config.items(): + curr_no_default_route = no_default_route + if topo_name in BGP_SCALE_T1S and 'spine' in v['properties']: + curr_no_default_route = True if dpus and k in dpus: continue vm_offset = vms[k]['vm_offset'] port = IPV4_BASE_PORT + vm_offset port6 = IPV6_BASE_PORT + vm_offset + routes_to_change[port] = [] + routes_to_change[port6] = [] aggregate_prefixes = v.get("aggregate_routes", AGGREGATE_ROUTES_DEFAULT_VALUE) aggregate_routes = [(prefix, nhipv4 if "." in prefix else nhipv6, "") for prefix in aggregate_prefixes] aggregate_routes_v4 = get_ipv4_routes(aggregate_routes) @@ -574,33 +614,80 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_defa tor_index = tornum - 1 if tornum is not None else None if router_type: if enable_ipv4_routes_generation: - routes_v4, _ = generate_routes("v4", podset_number, tor_number, tor_subnet_number, - None, leaf_asn_start, tor_asn_start, - nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1", - router_type=router_type, tor_index=tor_index, - no_default_route=no_default_route, tor_default_route=tor_default_route) + routes_v4, last_suffix = generate_routes("v4", podset_number, tor_number, tor_subnet_number, + None, leaf_asn_start, tor_asn_start, + nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1", + router_type=router_type, tor_index=tor_index, + no_default_route=curr_no_default_route, + tor_default_route=tor_default_route) if aggregate_routes_v4: filterout_subnet_ipv4(aggregate_routes, routes_v4) routes_v4.extend(aggregate_routes_v4) - change_routes(action, ptf_ip, port, routes_v4) + routes_to_change[port] += routes_v4 if enable_ipv6_routes_generation: - routes_v6, _ = generate_routes("v6", podset_number, tor_number, tor_subnet_number, - None, leaf_asn_start, tor_asn_start, - nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1", - router_type=router_type, tor_index=tor_index, - no_default_route=no_default_route, - ipv6_address_pattern=ipv6_address_pattern, - tor_default_route=tor_default_route) + routes_v6, last_suffix = generate_routes("v6", podset_number, tor_number, tor_subnet_number, + None, leaf_asn_start, tor_asn_start, + nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1", + router_type=router_type, tor_index=tor_index, + no_default_route=curr_no_default_route, + ipv6_address_pattern=ipv6_address_pattern, + tor_default_route=tor_default_route) if aggregate_routes_v6: filterout_subnet_ipv6(aggregate_routes, routes_v6) routes_v6.extend(aggregate_routes_v6) - change_routes(action, ptf_ip, port6, routes_v6) + routes_to_change[port6] += routes_v6 if 'vips' in v: routes_vips = [] for prefix in v["vips"]["ipv4"]["prefixes"]: routes_vips.append((prefix, nhipv4, v["vips"]["ipv4"]["asn"])) - change_routes(action, ptf_ip, port, routes_vips) + routes_to_change[port] += routes_vips + if topo_name in BGP_SCALE_T1S: + if downstream_neighbor_groups == 0: + downstream_neighbor_groups = common_config.get("downstream_neighbor_groups", DEFAULT_NEIGHBOR_GROUPS) + + # Announce T1 loopback received in T0 + leaf_number = common_config.get("leaf_number", LEAF_NUMBER) + tor_number = len([k for k, v in vms_config.items() if 'tor' in v['properties']]) + lov6_address_pattern = ipv6_address_pattern.split("/")[0] + "/128" + current_routes_offset = last_suffix + for index, (k, v) in enumerate(vms_config.items()): + if dpus and k in dpus: + continue + vm_offset = vms[k]['vm_offset'] + port = IPV4_BASE_PORT + vm_offset + port6 = IPV6_BASE_PORT + vm_offset + aggregate_prefixes = v.get("aggregate_routes", AGGREGATE_ROUTES_DEFAULT_VALUE) + aggregate_routes = [(prefix, nhipv4 if "." in prefix else nhipv6, "") for prefix in aggregate_prefixes] + aggregate_routes_v4 = get_ipv4_routes(aggregate_routes) + aggregate_routes_v6 = get_ipv6_routes(aggregate_routes) + if 'spine' in v['properties']: + continue + tor_asn = tor_asn_start + index + if enable_ipv4_routes_generation: + routes_v4, last_suffix = generate_t1_to_t0_routes("v4", current_routes_offset, leaf_number, 1, tor_asn, + leaf_asn_start, nhipv4, nhipv6, + ipv6_address_pattern=lov6_address_pattern) + if aggregate_routes_v4: + filterout_subnet_ipv4(aggregate_routes, routes_v4) + routes_v4.extend(aggregate_routes_v4) + routes_to_change[port] += routes_v4 + if enable_ipv6_routes_generation: + routes_v6, last_suffix = generate_t1_to_t0_routes("v6", current_routes_offset, leaf_number, 1, tor_asn, + leaf_asn_start, nhipv6, nhipv6, + ipv6_address_pattern=lov6_address_pattern) + if aggregate_routes_v6: + filterout_subnet_ipv6(aggregate_routes, routes_v6) + routes_v6.extend(aggregate_routes_v6) + routes_to_change[port6] += routes_v6 + group_index = index * downstream_neighbor_groups // tor_number + next_group_index = (index + 1) * downstream_neighbor_groups // tor_number + if group_index != next_group_index: + current_routes_offset += last_suffix + for port, routes in routes_to_change.items(): + if len(routes) <= 0: + continue + change_routes(action, ptf_ip, port, routes) def get_new_ip(curr_ip, skip_count): @@ -1194,7 +1281,8 @@ def main(): adhoc=dict(required=False, type='bool', default=False), peers_routes_to_change=dict(required=False, type='dict', default={}), log_path=dict(required=False, type='str', default=''), - upstream_neighbor_groups=dict(required=False, type='int', default=0) + upstream_neighbor_groups=dict(required=False, type='int', default=0), + downstream_neighbor_groups=dict(required=False, type='int', default=0) ), supports_check_mode=False) @@ -1209,6 +1297,7 @@ def main(): adhoc = module.params['adhoc'] peers_routes_to_change = module.params['peers_routes_to_change'] upstream_neighbor_groups = module.params['upstream_neighbor_groups'] + downstream_neighbor_groups = module.params['downstream_neighbor_groups'] topo = read_topo(topo_name, path) if not topo: @@ -1234,7 +1323,8 @@ def main(): module.exit_json(changed=True) elif topo_type == "t1" or topo_type == "smartswitch-t1": fib_t1_lag( - topo, ptf_ip, no_default_route=is_storage_backend, action=action, tor_default_route=tor_default_route) + topo, ptf_ip, topo_name, no_default_route=is_storage_backend, action=action, + tor_default_route=tor_default_route, downstream_neighbor_groups=downstream_neighbor_groups) module.exit_json(changed=True) elif topo_type == "t2": fib_t2_lag(topo, ptf_ip, action=action) diff --git a/ansible/roles/vm_set/tasks/announce_routes.yml b/ansible/roles/vm_set/tasks/announce_routes.yml index 5e2137f306..69aadee758 100644 --- a/ansible/roles/vm_set/tasks/announce_routes.yml +++ b/ansible/roles/vm_set/tasks/announce_routes.yml @@ -127,6 +127,7 @@ topo_name: "{{ topo }}" ptf_ip: "{{ ptf_host_ip }}" dut_interfaces: "{{ dut_interfaces | default('') }}" - upstream_neighbor_groups: "{{ upstream_neighbor_groups | int }}" + upstream_neighbor_groups: "{{ upstream_neighbor_groups | default(0) | int }}" + downstream_neighbor_groups: "{{ downstream_neighbor_groups | default(0) | int }}" delegate_to: localhost when: exabgp_action == 'start' diff --git a/ansible/testbed-cli.sh b/ansible/testbed-cli.sh index 19a9fd7842..acd476291a 100755 --- a/ansible/testbed-cli.sh +++ b/ansible/testbed-cli.sh @@ -142,7 +142,7 @@ function read_yaml tb_line=${tb_lines[0]} line_arr=($1) - for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 ptf_extra_mgmt_ip netns_mgmt_ip server vm_base dut inv_name auto_recover comment servers upstream_neighbor_groups; + for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 ptf_extra_mgmt_ip netns_mgmt_ip server vm_base dut inv_name auto_recover comment servers upstream_neighbor_groups downstream_neighbor_groups; do value=$(python -c "from __future__ import print_function; tb=eval(\"$tb_line\"); print(tb.get('$attr', None))") [ "$value" == "None" ] && value= @@ -167,6 +167,7 @@ function read_yaml inv_name=${line_arr[12]} servers=${line_arr[15]} upstream_neighbor_groups=${line_arr[16]} + downstream_neighbor_groups=${line_arr[17]} # Remove the dpu duts by the keyword 'dpu' in the dut name duts=$(echo $duts | sed "s/,[^,]*dpu[^,]*//g") } @@ -320,7 +321,7 @@ function add_topo -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$vm_set_name" \ -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" -e ptf_ipv6="$ptf_ipv6" \ -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" -e netns_mgmt_ip="$netns_mgmt_ip" \ - -e upstream_neighbor_groups="$upstream_neighbor_groups" \ + -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \ $ansible_options $@ if [ $i -eq 0 ]; then @@ -438,7 +439,7 @@ function renumber_topo ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_renumber_vm_topology.yml --vault-password-file="${passwd}" \ -l "$server" -e testbed_name="$testbed_name" -e duts_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" \ -e topo="$topo" -e vm_set_name="$vm_set_name" -e ptf_imagename="$ptf_imagename" -e ptf_ipv6="$ptf_ipv6" \ - -e upstream_neighbor_groups="$upstream_neighbor_groups" \ + -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \ -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" $@ ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="${passwd}" -e "dut=$duts" $@ @@ -488,7 +489,7 @@ function refresh_dut -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$vm_set_name" \ -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" -e ptf_ipv6="$ptf_ipv6" \ -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" -e force_stop_sonic_vm="yes" \ - -e upstream_neighbor_groups="$upstream_neighbor_groups" \ + -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \ $ansible_options $@ echo Done diff --git a/ansible/vars/topo_t1-isolated-d254u2.yml b/ansible/vars/topo_t1-isolated-d254u2.yml index 3d215ab176..0d0afeb38e 100644 --- a/ansible/vars/topo_t1-isolated-d254u2.yml +++ b/ansible/vars/topo_t1-isolated-d254u2.yml @@ -1038,6 +1038,7 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 256 spine: swrole: spine tor: diff --git a/ansible/vars/topo_t1-isolated-d254u2s1.yml b/ansible/vars/topo_t1-isolated-d254u2s1.yml index ebd6c5b4f3..fac88b2ff8 100644 --- a/ansible/vars/topo_t1-isolated-d254u2s1.yml +++ b/ansible/vars/topo_t1-isolated-d254u2s1.yml @@ -1042,6 +1042,7 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 256 spine: swrole: spine tor: diff --git a/ansible/vars/topo_t1-isolated-d254u2s2.yml b/ansible/vars/topo_t1-isolated-d254u2s2.yml index feeae03202..1f8597d14b 100644 --- a/ansible/vars/topo_t1-isolated-d254u2s2.yml +++ b/ansible/vars/topo_t1-isolated-d254u2s2.yml @@ -1046,6 +1046,7 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 256 spine: swrole: spine tor: diff --git a/ansible/vars/topo_t1-isolated-d510u2.yml b/ansible/vars/topo_t1-isolated-d510u2.yml index 925142e04c..7b1124f893 100644 --- a/ansible/vars/topo_t1-isolated-d510u2.yml +++ b/ansible/vars/topo_t1-isolated-d510u2.yml @@ -2062,6 +2062,8 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 512 + downstream_neighbor_groups: 2 spine: swrole: spine tor: diff --git a/ansible/vars/topo_t1-isolated-d510u2s2.yml b/ansible/vars/topo_t1-isolated-d510u2s2.yml index 0f907a7f82..89e43a5e50 100644 --- a/ansible/vars/topo_t1-isolated-d510u2s2.yml +++ b/ansible/vars/topo_t1-isolated-d510u2s2.yml @@ -2070,6 +2070,8 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 256 + downstream_neighbor_groups: 2 spine: swrole: spine tor: diff --git a/tests/bgp/test_ipv6_bgp_scale.py b/tests/bgp/test_ipv6_bgp_scale.py index 1bbefbd669..7858bb1266 100644 --- a/tests/bgp/test_ipv6_bgp_scale.py +++ b/tests/bgp/test_ipv6_bgp_scale.py @@ -19,7 +19,8 @@ pytestmark = [ pytest.mark.topology( 't0-isolated-d2u254s1', 't0-isolated-d2u254s2', 't0-isolated-d2u510', - 't1-isolated-d254u2s1', 't1-isolated-d254u2s2', 't1-isolated-d510u2' + 't1-isolated-d254u2s1', 't1-isolated-d254u2s2', 't1-isolated-d510u2', + 't1-isolated-d254u2', 't1-isolated-d510u2s2' ) ] @@ -131,7 +132,8 @@ def announce_routes(localhost, tbinfo, ptf_ip, dut_interfaces): path="../ansible/", log_path="logs", dut_interfaces=dut_interfaces, - upstream_neighbor_groups=tbinfo['upstream_neighbor_groups'] if 'upstream_neighbor_groups' in tbinfo else None + upstream_neighbor_groups=tbinfo['upstream_neighbor_groups'] if 'upstream_neighbor_groups' in tbinfo else 0, + downstream_neighbor_groups=tbinfo['downstream_neighbor_groups'] if 'downstream_neighbor_groups' in tbinfo else 0 ) @@ -144,7 +146,8 @@ def get_all_bgp_ipv6_routes(duthost): def generate_packets(prefixes, dut_mac, src_mac): pkts = [] for prefix in prefixes: - addr = str(ipaddress.ip_network(prefix)[1]) + network = ipaddress.ip_network(prefix) + addr = str(network[0] if network.num_addresses == 1 else network[1]) pkt = simple_icmpv6_packet( eth_dst=dut_mac, eth_src=src_mac, @@ -514,7 +517,8 @@ def test_device_unisolation( ptfadapter, bgp_peers_info, setup_packet_mask_counters, - announce_bgp_routes_teardown + announce_bgp_routes_teardown, + tbinfo ): ''' This test is for the worst senario that all ports are flapped,