From 488e3540af8cd92ea601774600725789c0e71311 Mon Sep 17 00:00:00 2001 From: Sonic Build Admin Date: Mon, 23 Jun 2025 11:12:22 +0000 Subject: [PATCH] [bgp_scale] Fix announce_routes for t1-isolated topo ### Description of PR Summary: Fixes # (issue) ### Type of change - [ ] Bug fix - [ ] Testbed and Framework(new/improvement) - [ ] New Test case - [ ] Skipped for non-supported platforms - [ ] Test case improvement ### Back port request - [ ] 202205 - [ ] 202305 - [ ] 202311 - [ ] 202405 - [ ] 202411 - [ ] 202505 ### Approach #### What is the motivation for this PR? Fix incorrect route announcing in isolated T1 topos #### How did you do it? 1. For announce_routes.py, update to announce correct routes - Announce more routes, details: https://github.com/sonic-net/sonic-mgmt/pull/19128 - Add support to announce different sets from downstream 3. For test_ipv6_bgp_scale, because T0 and T2 would both announce default routes, but T0s' has the shorter as path. Hence by default there are only default routes from T0s taking effect. In test, after shutdown all T0 neighbor ports, default routes wouldn't disappear, but default routes from T2 would appear. Hence update routes verification here. #### How did you verify/test it? Run tests #### Any platform specific information? #### Supported testbed topology if it's a new test case? ### Documentation --- ansible/library/announce_routes.py | 126 +++++++++++++++--- .../roles/vm_set/tasks/announce_routes.yml | 3 +- ansible/testbed-cli.sh | 9 +- ansible/vars/topo_t1-isolated-d254u2.yml | 1 + ansible/vars/topo_t1-isolated-d254u2s1.yml | 1 + ansible/vars/topo_t1-isolated-d254u2s2.yml | 1 + ansible/vars/topo_t1-isolated-d510u2.yml | 2 + ansible/vars/topo_t1-isolated-d510u2s2.yml | 2 + tests/bgp/test_ipv6_bgp_scale.py | 12 +- 9 files changed, 130 insertions(+), 27 deletions(-) diff --git a/ansible/library/announce_routes.py b/ansible/library/announce_routes.py index 19f94c87f7..4b78de4f84 100644 --- a/ansible/library/announce_routes.py +++ b/ansible/library/announce_routes.py @@ -75,6 +75,10 @@ IPV6_ADDRESS_PATTERN_DEFAULT_VALUE = '20%02X:%02X%02X:0:%02X::/64' ENABLE_IPV4_ROUTES_GENERATION_DEFAULT_VALUE = True ENABLE_IPV6_ROUTES_GENERATION_DEFAULT_VALUE = True +BGP_SCALE_T1S = [ + 't1-isolated-d254u2', 't1-isolated-d254u2s1', 't1-isolated-d254u2s2', + 't1-isolated-d510u2', 't1-isolated-d510u2s2' +] # Describe default number of COLOs COLO_NUMBER = 30 @@ -100,6 +104,8 @@ M0_SUBNET_PREFIX_LEN_V6 = 64 # Describe default start asn of M1s M1_ASN_START = 65200 +# Describe default leaf number +LEAF_NUMBER = 256 def wait_for_http(host_ip, http_port, timeout=10): @@ -461,6 +467,30 @@ def generate_routes(family, podset_number, tor_number, tor_subnet_number, return routes, suffix +def generate_t1_to_t0_routes(family, offset, leaf_number, subnet_size, tor_asn, leaf_asn_start, nexthop, nexthop_v6, + podset_num=1, ipv6_address_pattern=IPV6_ADDRESS_PATTERN_DEFAULT_VALUE): + routes = [] + for podset in range(0, podset_num): + for leaf in range(0, leaf_number): + suffix = offset + leaf + octet2 = (168 + int(suffix / (256 ** 2))) + octet1 = (192 + int(octet2 / 256)) + octet2 = (octet2 % 256) + octet3 = (int(suffix / 256) % 256) + octet4 = (suffix % 256) + prefixlen_v4 = (32 - int(math.log(subnet_size, 2))) + prefix = "{}.{}.{}.{}/{}".format(octet1, octet2, octet3, octet4, prefixlen_v4) + prefix_v6 = ipv6_address_pattern % ( + octet1, octet2, octet3, octet4) + leaf_asn = leaf_asn_start + podset + aspath = "{} {}".format(leaf_asn, tor_asn) + if family in ["v4", "both"]: + routes.append((prefix, nexthop, aspath)) + if family in ["v6", "both"]: + routes.append((prefix_v6, nexthop_v6, aspath)) + return routes, suffix + + def fib_t0(topo, ptf_ip, no_default_route=False, action="announce", upstream_neighbor_groups=0): common_config = topo['configuration_properties'].get('common', {}) podset_number = common_config.get("podset_number", PODSET_NUMBER) @@ -527,7 +557,8 @@ def fib_t0(topo, ptf_ip, no_default_route=False, action="announce", upstream_nei current_routes_offset += last_suffix -def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_default_route=False): +def fib_t1_lag(topo, ptf_ip, topo_name, no_default_route=False, action="announce", tor_default_route=False, + downstream_neighbor_groups=0): common_config = topo['configuration_properties'].get('common', {}) podset_number = common_config.get("podset_number", PODSET_NUMBER) tor_number = common_config.get("tor_number", TOR_NUMBER) @@ -553,13 +584,22 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_defa if 'DPUs' in topo['topology']: dpus = topo['topology']['DPUs'] + last_suffix = 0 + if topo_name in BGP_SCALE_T1S: + tor_default_route = True + routes_to_change = {} for k, v in vms_config.items(): + curr_no_default_route = no_default_route + if topo_name in BGP_SCALE_T1S and 'spine' in v['properties']: + curr_no_default_route = True if dpus and k in dpus: continue vm_offset = vms[k]['vm_offset'] port = IPV4_BASE_PORT + vm_offset port6 = IPV6_BASE_PORT + vm_offset + routes_to_change[port] = [] + routes_to_change[port6] = [] aggregate_prefixes = v.get("aggregate_routes", AGGREGATE_ROUTES_DEFAULT_VALUE) aggregate_routes = [(prefix, nhipv4 if "." in prefix else nhipv6, "") for prefix in aggregate_prefixes] aggregate_routes_v4 = get_ipv4_routes(aggregate_routes) @@ -574,33 +614,80 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_defa tor_index = tornum - 1 if tornum is not None else None if router_type: if enable_ipv4_routes_generation: - routes_v4, _ = generate_routes("v4", podset_number, tor_number, tor_subnet_number, - None, leaf_asn_start, tor_asn_start, - nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1", - router_type=router_type, tor_index=tor_index, - no_default_route=no_default_route, tor_default_route=tor_default_route) + routes_v4, last_suffix = generate_routes("v4", podset_number, tor_number, tor_subnet_number, + None, leaf_asn_start, tor_asn_start, + nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1", + router_type=router_type, tor_index=tor_index, + no_default_route=curr_no_default_route, + tor_default_route=tor_default_route) if aggregate_routes_v4: filterout_subnet_ipv4(aggregate_routes, routes_v4) routes_v4.extend(aggregate_routes_v4) - change_routes(action, ptf_ip, port, routes_v4) + routes_to_change[port] += routes_v4 if enable_ipv6_routes_generation: - routes_v6, _ = generate_routes("v6", podset_number, tor_number, tor_subnet_number, - None, leaf_asn_start, tor_asn_start, - nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1", - router_type=router_type, tor_index=tor_index, - no_default_route=no_default_route, - ipv6_address_pattern=ipv6_address_pattern, - tor_default_route=tor_default_route) + routes_v6, last_suffix = generate_routes("v6", podset_number, tor_number, tor_subnet_number, + None, leaf_asn_start, tor_asn_start, + nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1", + router_type=router_type, tor_index=tor_index, + no_default_route=curr_no_default_route, + ipv6_address_pattern=ipv6_address_pattern, + tor_default_route=tor_default_route) if aggregate_routes_v6: filterout_subnet_ipv6(aggregate_routes, routes_v6) routes_v6.extend(aggregate_routes_v6) - change_routes(action, ptf_ip, port6, routes_v6) + routes_to_change[port6] += routes_v6 if 'vips' in v: routes_vips = [] for prefix in v["vips"]["ipv4"]["prefixes"]: routes_vips.append((prefix, nhipv4, v["vips"]["ipv4"]["asn"])) - change_routes(action, ptf_ip, port, routes_vips) + routes_to_change[port] += routes_vips + if topo_name in BGP_SCALE_T1S: + if downstream_neighbor_groups == 0: + downstream_neighbor_groups = common_config.get("downstream_neighbor_groups", DEFAULT_NEIGHBOR_GROUPS) + + # Announce T1 loopback received in T0 + leaf_number = common_config.get("leaf_number", LEAF_NUMBER) + tor_number = len([k for k, v in vms_config.items() if 'tor' in v['properties']]) + lov6_address_pattern = ipv6_address_pattern.split("/")[0] + "/128" + current_routes_offset = last_suffix + for index, (k, v) in enumerate(vms_config.items()): + if dpus and k in dpus: + continue + vm_offset = vms[k]['vm_offset'] + port = IPV4_BASE_PORT + vm_offset + port6 = IPV6_BASE_PORT + vm_offset + aggregate_prefixes = v.get("aggregate_routes", AGGREGATE_ROUTES_DEFAULT_VALUE) + aggregate_routes = [(prefix, nhipv4 if "." in prefix else nhipv6, "") for prefix in aggregate_prefixes] + aggregate_routes_v4 = get_ipv4_routes(aggregate_routes) + aggregate_routes_v6 = get_ipv6_routes(aggregate_routes) + if 'spine' in v['properties']: + continue + tor_asn = tor_asn_start + index + if enable_ipv4_routes_generation: + routes_v4, last_suffix = generate_t1_to_t0_routes("v4", current_routes_offset, leaf_number, 1, tor_asn, + leaf_asn_start, nhipv4, nhipv6, + ipv6_address_pattern=lov6_address_pattern) + if aggregate_routes_v4: + filterout_subnet_ipv4(aggregate_routes, routes_v4) + routes_v4.extend(aggregate_routes_v4) + routes_to_change[port] += routes_v4 + if enable_ipv6_routes_generation: + routes_v6, last_suffix = generate_t1_to_t0_routes("v6", current_routes_offset, leaf_number, 1, tor_asn, + leaf_asn_start, nhipv6, nhipv6, + ipv6_address_pattern=lov6_address_pattern) + if aggregate_routes_v6: + filterout_subnet_ipv6(aggregate_routes, routes_v6) + routes_v6.extend(aggregate_routes_v6) + routes_to_change[port6] += routes_v6 + group_index = index * downstream_neighbor_groups // tor_number + next_group_index = (index + 1) * downstream_neighbor_groups // tor_number + if group_index != next_group_index: + current_routes_offset += last_suffix + for port, routes in routes_to_change.items(): + if len(routes) <= 0: + continue + change_routes(action, ptf_ip, port, routes) def get_new_ip(curr_ip, skip_count): @@ -1194,7 +1281,8 @@ def main(): adhoc=dict(required=False, type='bool', default=False), peers_routes_to_change=dict(required=False, type='dict', default={}), log_path=dict(required=False, type='str', default=''), - upstream_neighbor_groups=dict(required=False, type='int', default=0) + upstream_neighbor_groups=dict(required=False, type='int', default=0), + downstream_neighbor_groups=dict(required=False, type='int', default=0) ), supports_check_mode=False) @@ -1209,6 +1297,7 @@ def main(): adhoc = module.params['adhoc'] peers_routes_to_change = module.params['peers_routes_to_change'] upstream_neighbor_groups = module.params['upstream_neighbor_groups'] + downstream_neighbor_groups = module.params['downstream_neighbor_groups'] topo = read_topo(topo_name, path) if not topo: @@ -1234,7 +1323,8 @@ def main(): module.exit_json(changed=True) elif topo_type == "t1" or topo_type == "smartswitch-t1": fib_t1_lag( - topo, ptf_ip, no_default_route=is_storage_backend, action=action, tor_default_route=tor_default_route) + topo, ptf_ip, topo_name, no_default_route=is_storage_backend, action=action, + tor_default_route=tor_default_route, downstream_neighbor_groups=downstream_neighbor_groups) module.exit_json(changed=True) elif topo_type == "t2": fib_t2_lag(topo, ptf_ip, action=action) diff --git a/ansible/roles/vm_set/tasks/announce_routes.yml b/ansible/roles/vm_set/tasks/announce_routes.yml index 5e2137f306..69aadee758 100644 --- a/ansible/roles/vm_set/tasks/announce_routes.yml +++ b/ansible/roles/vm_set/tasks/announce_routes.yml @@ -127,6 +127,7 @@ topo_name: "{{ topo }}" ptf_ip: "{{ ptf_host_ip }}" dut_interfaces: "{{ dut_interfaces | default('') }}" - upstream_neighbor_groups: "{{ upstream_neighbor_groups | int }}" + upstream_neighbor_groups: "{{ upstream_neighbor_groups | default(0) | int }}" + downstream_neighbor_groups: "{{ downstream_neighbor_groups | default(0) | int }}" delegate_to: localhost when: exabgp_action == 'start' diff --git a/ansible/testbed-cli.sh b/ansible/testbed-cli.sh index 19a9fd7842..acd476291a 100755 --- a/ansible/testbed-cli.sh +++ b/ansible/testbed-cli.sh @@ -142,7 +142,7 @@ function read_yaml tb_line=${tb_lines[0]} line_arr=($1) - for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 ptf_extra_mgmt_ip netns_mgmt_ip server vm_base dut inv_name auto_recover comment servers upstream_neighbor_groups; + for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 ptf_extra_mgmt_ip netns_mgmt_ip server vm_base dut inv_name auto_recover comment servers upstream_neighbor_groups downstream_neighbor_groups; do value=$(python -c "from __future__ import print_function; tb=eval(\"$tb_line\"); print(tb.get('$attr', None))") [ "$value" == "None" ] && value= @@ -167,6 +167,7 @@ function read_yaml inv_name=${line_arr[12]} servers=${line_arr[15]} upstream_neighbor_groups=${line_arr[16]} + downstream_neighbor_groups=${line_arr[17]} # Remove the dpu duts by the keyword 'dpu' in the dut name duts=$(echo $duts | sed "s/,[^,]*dpu[^,]*//g") } @@ -320,7 +321,7 @@ function add_topo -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$vm_set_name" \ -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" -e ptf_ipv6="$ptf_ipv6" \ -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" -e netns_mgmt_ip="$netns_mgmt_ip" \ - -e upstream_neighbor_groups="$upstream_neighbor_groups" \ + -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \ $ansible_options $@ if [ $i -eq 0 ]; then @@ -438,7 +439,7 @@ function renumber_topo ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_renumber_vm_topology.yml --vault-password-file="${passwd}" \ -l "$server" -e testbed_name="$testbed_name" -e duts_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" \ -e topo="$topo" -e vm_set_name="$vm_set_name" -e ptf_imagename="$ptf_imagename" -e ptf_ipv6="$ptf_ipv6" \ - -e upstream_neighbor_groups="$upstream_neighbor_groups" \ + -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \ -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" $@ ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="${passwd}" -e "dut=$duts" $@ @@ -488,7 +489,7 @@ function refresh_dut -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$vm_set_name" \ -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" -e ptf_ipv6="$ptf_ipv6" \ -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" -e force_stop_sonic_vm="yes" \ - -e upstream_neighbor_groups="$upstream_neighbor_groups" \ + -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \ $ansible_options $@ echo Done diff --git a/ansible/vars/topo_t1-isolated-d254u2.yml b/ansible/vars/topo_t1-isolated-d254u2.yml index 3d215ab176..0d0afeb38e 100644 --- a/ansible/vars/topo_t1-isolated-d254u2.yml +++ b/ansible/vars/topo_t1-isolated-d254u2.yml @@ -1038,6 +1038,7 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 256 spine: swrole: spine tor: diff --git a/ansible/vars/topo_t1-isolated-d254u2s1.yml b/ansible/vars/topo_t1-isolated-d254u2s1.yml index ebd6c5b4f3..fac88b2ff8 100644 --- a/ansible/vars/topo_t1-isolated-d254u2s1.yml +++ b/ansible/vars/topo_t1-isolated-d254u2s1.yml @@ -1042,6 +1042,7 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 256 spine: swrole: spine tor: diff --git a/ansible/vars/topo_t1-isolated-d254u2s2.yml b/ansible/vars/topo_t1-isolated-d254u2s2.yml index feeae03202..1f8597d14b 100644 --- a/ansible/vars/topo_t1-isolated-d254u2s2.yml +++ b/ansible/vars/topo_t1-isolated-d254u2s2.yml @@ -1046,6 +1046,7 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 256 spine: swrole: spine tor: diff --git a/ansible/vars/topo_t1-isolated-d510u2.yml b/ansible/vars/topo_t1-isolated-d510u2.yml index 925142e04c..7b1124f893 100644 --- a/ansible/vars/topo_t1-isolated-d510u2.yml +++ b/ansible/vars/topo_t1-isolated-d510u2.yml @@ -2062,6 +2062,8 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 512 + downstream_neighbor_groups: 2 spine: swrole: spine tor: diff --git a/ansible/vars/topo_t1-isolated-d510u2s2.yml b/ansible/vars/topo_t1-isolated-d510u2s2.yml index 0f907a7f82..89e43a5e50 100644 --- a/ansible/vars/topo_t1-isolated-d510u2s2.yml +++ b/ansible/vars/topo_t1-isolated-d510u2s2.yml @@ -2070,6 +2070,8 @@ configuration_properties: ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120 enable_ipv4_routes_generation: false enable_ipv6_routes_generation: true + leaf_number: 256 + downstream_neighbor_groups: 2 spine: swrole: spine tor: diff --git a/tests/bgp/test_ipv6_bgp_scale.py b/tests/bgp/test_ipv6_bgp_scale.py index 1bbefbd669..7858bb1266 100644 --- a/tests/bgp/test_ipv6_bgp_scale.py +++ b/tests/bgp/test_ipv6_bgp_scale.py @@ -19,7 +19,8 @@ pytestmark = [ pytest.mark.topology( 't0-isolated-d2u254s1', 't0-isolated-d2u254s2', 't0-isolated-d2u510', - 't1-isolated-d254u2s1', 't1-isolated-d254u2s2', 't1-isolated-d510u2' + 't1-isolated-d254u2s1', 't1-isolated-d254u2s2', 't1-isolated-d510u2', + 't1-isolated-d254u2', 't1-isolated-d510u2s2' ) ] @@ -131,7 +132,8 @@ def announce_routes(localhost, tbinfo, ptf_ip, dut_interfaces): path="../ansible/", log_path="logs", dut_interfaces=dut_interfaces, - upstream_neighbor_groups=tbinfo['upstream_neighbor_groups'] if 'upstream_neighbor_groups' in tbinfo else None + upstream_neighbor_groups=tbinfo['upstream_neighbor_groups'] if 'upstream_neighbor_groups' in tbinfo else 0, + downstream_neighbor_groups=tbinfo['downstream_neighbor_groups'] if 'downstream_neighbor_groups' in tbinfo else 0 ) @@ -144,7 +146,8 @@ def get_all_bgp_ipv6_routes(duthost): def generate_packets(prefixes, dut_mac, src_mac): pkts = [] for prefix in prefixes: - addr = str(ipaddress.ip_network(prefix)[1]) + network = ipaddress.ip_network(prefix) + addr = str(network[0] if network.num_addresses == 1 else network[1]) pkt = simple_icmpv6_packet( eth_dst=dut_mac, eth_src=src_mac, @@ -514,7 +517,8 @@ def test_device_unisolation( ptfadapter, bgp_peers_info, setup_packet_mask_counters, - announce_bgp_routes_teardown + announce_bgp_routes_teardown, + tbinfo ): ''' This test is for the worst senario that all ports are flapped,