[bgp_scale] Fix announce_routes for t1-isolated topo (sonic-net#19117)

What is the motivation for this PR?
Fix incorrect route announcing in isolated T1 topos

How did you do it?
For announce_routes.py, update to announce correct routes
Announce more routes, details: [doc] Update announce_routes doc for isolated T1 sonic-net#19128
Add support to announce different sets from downstream
For test_ipv6_bgp_scale, because T0 and T2 would both announce default routes, but T0s' has the shorter as path. Hence by default there are only default routes from T0s taking effect. In test, after shutdown all T0 neighbor ports, default routes wouldn't disappear, but default routes from T2 would appear. Hence update routes verification here.

How did you verify/test it?
Run tests

Signed-off-by: Aharon Malkin <[email protected]>

Author: yaqiangz

ansible/library/announce_routes.py

@@ -76,6 +76,10 @@
 IPV6_ADDRESS_PATTERN_DEFAULT_VALUE = '20%02X:%02X%02X:0:%02X::/64'
 ENABLE_IPV4_ROUTES_GENERATION_DEFAULT_VALUE = True
 ENABLE_IPV6_ROUTES_GENERATION_DEFAULT_VALUE = True
+BGP_SCALE_T1S = [
+    't1-isolated-d254u2', 't1-isolated-d254u2s1', 't1-isolated-d254u2s2',
+    't1-isolated-d510u2', 't1-isolated-d510u2s2'
+]
 
 # Describe default number of COLOs
 COLO_NUMBER = 30
@@ -105,6 +109,8 @@
 M0_SUBNET_PREFIX_LEN_V6 = 64
 # Describe default start asn of M1s
 M1_ASN_START = 65200
+# Describe default leaf number
+LEAF_NUMBER = 256
 
 
 def wait_for_http(host_ip, http_port, timeout=10):
@@ -481,6 +487,30 @@ def generate_routes(family, podset_number, tor_number, tor_subnet_number,
     return routes, suffix
 
 
+def generate_t1_to_t0_routes(family, offset, leaf_number, subnet_size, tor_asn, leaf_asn_start, nexthop, nexthop_v6,
+                             podset_num=1, ipv6_address_pattern=IPV6_ADDRESS_PATTERN_DEFAULT_VALUE):
+    routes = []
+    for podset in range(0, podset_num):
+        for leaf in range(0, leaf_number):
+            suffix = offset + leaf
+            octet2 = (168 + int(suffix / (256 ** 2)))
+            octet1 = (192 + int(octet2 / 256))
+            octet2 = (octet2 % 256)
+            octet3 = (int(suffix / 256) % 256)
+            octet4 = (suffix % 256)
+            prefixlen_v4 = (32 - int(math.log(subnet_size, 2)))
+            prefix = "{}.{}.{}.{}/{}".format(octet1, octet2, octet3, octet4, prefixlen_v4)
+            prefix_v6 = ipv6_address_pattern % (
+                octet1, octet2, octet3, octet4)
+            leaf_asn = leaf_asn_start + podset
+            aspath = "{} {}".format(leaf_asn, tor_asn)
+            if family in ["v4", "both"]:
+                routes.append((prefix, nexthop, aspath))
+            if family in ["v6", "both"]:
+                routes.append((prefix_v6, nexthop_v6, aspath))
+    return routes, suffix
+
+
 def fib_t0(topo, ptf_ip, no_default_route=False, action="announce", upstream_neighbor_groups=0):
     common_config = topo['configuration_properties'].get('common', {})
     podset_number = common_config.get("podset_number", PODSET_NUMBER)
@@ -547,7 +577,8 @@ def fib_t0(topo, ptf_ip, no_default_route=False, action="announce", upstream_nei
             current_routes_offset += last_suffix
 
 
-def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_default_route=False):
+def fib_t1_lag(topo, ptf_ip, topo_name, no_default_route=False, action="announce", tor_default_route=False,
+               downstream_neighbor_groups=0):
     common_config = topo['configuration_properties'].get('common', {})
     podset_number = common_config.get("podset_number", PODSET_NUMBER)
     tor_number = common_config.get("tor_number", TOR_NUMBER)
@@ -573,13 +604,22 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_defa
     if 'DPUs' in topo['topology']:
         dpus = topo['topology']['DPUs']
 
+    last_suffix = 0
+    if topo_name in BGP_SCALE_T1S:
+        tor_default_route = True
+    routes_to_change = {}
     for k, v in vms_config.items():
+        curr_no_default_route = no_default_route
+        if topo_name in BGP_SCALE_T1S and 'spine' in v['properties']:
+            curr_no_default_route = True
         if dpus and k in dpus:
             continue
 
         vm_offset = vms[k]['vm_offset']
         port = IPV4_BASE_PORT + vm_offset
         port6 = IPV6_BASE_PORT + vm_offset
+        routes_to_change[port] = []
+        routes_to_change[port6] = []
         aggregate_prefixes = v.get("aggregate_routes", AGGREGATE_ROUTES_DEFAULT_VALUE)
         aggregate_routes = [(prefix, nhipv4 if "." in prefix else nhipv6, "") for prefix in aggregate_prefixes]
         aggregate_routes_v4 = get_ipv4_routes(aggregate_routes)
@@ -594,33 +634,80 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_defa
         tor_index = tornum - 1 if tornum is not None else None
         if router_type:
             if enable_ipv4_routes_generation:
-                routes_v4, _ = generate_routes("v4", podset_number, tor_number, tor_subnet_number,
-                                               None, leaf_asn_start, tor_asn_start,
-                                               nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
-                                               router_type=router_type, tor_index=tor_index,
-                                               no_default_route=no_default_route, tor_default_route=tor_default_route)
+                routes_v4, last_suffix = generate_routes("v4", podset_number, tor_number, tor_subnet_number,
+                                                         None, leaf_asn_start, tor_asn_start,
+                                                         nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
+                                                         router_type=router_type, tor_index=tor_index,
+                                                         no_default_route=curr_no_default_route,
+                                                         tor_default_route=tor_default_route)
                 if aggregate_routes_v4:
                     filterout_subnet_ipv4(aggregate_routes, routes_v4)
                     routes_v4.extend(aggregate_routes_v4)
-                change_routes(action, ptf_ip, port, routes_v4)
+                routes_to_change[port] += routes_v4
             if enable_ipv6_routes_generation:
-                routes_v6, _ = generate_routes("v6", podset_number, tor_number, tor_subnet_number,
-                                               None, leaf_asn_start, tor_asn_start,
-                                               nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
-                                               router_type=router_type, tor_index=tor_index,
-                                               no_default_route=no_default_route,
-                                               ipv6_address_pattern=ipv6_address_pattern,
-                                               tor_default_route=tor_default_route)
+                routes_v6, last_suffix = generate_routes("v6", podset_number, tor_number, tor_subnet_number,
+                                                         None, leaf_asn_start, tor_asn_start,
+                                                         nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
+                                                         router_type=router_type, tor_index=tor_index,
+                                                         no_default_route=curr_no_default_route,
+                                                         ipv6_address_pattern=ipv6_address_pattern,
+                                                         tor_default_route=tor_default_route)
                 if aggregate_routes_v6:
                     filterout_subnet_ipv6(aggregate_routes, routes_v6)
                     routes_v6.extend(aggregate_routes_v6)
-                change_routes(action, ptf_ip, port6, routes_v6)
+                routes_to_change[port6] += routes_v6
 
         if 'vips' in v:
             routes_vips = []
             for prefix in v["vips"]["ipv4"]["prefixes"]:
                 routes_vips.append((prefix, nhipv4, v["vips"]["ipv4"]["asn"]))
-            change_routes(action, ptf_ip, port, routes_vips)
+            routes_to_change[port] += routes_vips
+    if topo_name in BGP_SCALE_T1S:
+        if downstream_neighbor_groups == 0:
+            downstream_neighbor_groups = common_config.get("downstream_neighbor_groups", DEFAULT_NEIGHBOR_GROUPS)
+
+        # Announce T1 loopback received in T0
+        leaf_number = common_config.get("leaf_number", LEAF_NUMBER)
+        tor_number = len([k for k, v in vms_config.items() if 'tor' in v['properties']])
+        lov6_address_pattern = ipv6_address_pattern.split("/")[0] + "/128"
+        current_routes_offset = last_suffix
+        for index, (k, v) in enumerate(vms_config.items()):
+            if dpus and k in dpus:
+                continue
+            vm_offset = vms[k]['vm_offset']
+            port = IPV4_BASE_PORT + vm_offset
+            port6 = IPV6_BASE_PORT + vm_offset
+            aggregate_prefixes = v.get("aggregate_routes", AGGREGATE_ROUTES_DEFAULT_VALUE)
+            aggregate_routes = [(prefix, nhipv4 if "." in prefix else nhipv6, "") for prefix in aggregate_prefixes]
+            aggregate_routes_v4 = get_ipv4_routes(aggregate_routes)
+            aggregate_routes_v6 = get_ipv6_routes(aggregate_routes)
+            if 'spine' in v['properties']:
+                continue
+            tor_asn = tor_asn_start + index
+            if enable_ipv4_routes_generation:
+                routes_v4, last_suffix = generate_t1_to_t0_routes("v4", current_routes_offset, leaf_number, 1, tor_asn,
+                                                                  leaf_asn_start, nhipv4, nhipv6,
+                                                                  ipv6_address_pattern=lov6_address_pattern)
+                if aggregate_routes_v4:
+                    filterout_subnet_ipv4(aggregate_routes, routes_v4)
+                    routes_v4.extend(aggregate_routes_v4)
+                routes_to_change[port] += routes_v4
+            if enable_ipv6_routes_generation:
+                routes_v6, last_suffix = generate_t1_to_t0_routes("v6", current_routes_offset, leaf_number, 1, tor_asn,
+                                                                  leaf_asn_start, nhipv6, nhipv6,
+                                                                  ipv6_address_pattern=lov6_address_pattern)
+                if aggregate_routes_v6:
+                    filterout_subnet_ipv6(aggregate_routes, routes_v6)
+                    routes_v6.extend(aggregate_routes_v6)
+                routes_to_change[port6] += routes_v6
+            group_index = index * downstream_neighbor_groups // tor_number
+            next_group_index = (index + 1) * downstream_neighbor_groups // tor_number
+            if group_index != next_group_index:
+                current_routes_offset += last_suffix
+    for port, routes in routes_to_change.items():
+        if len(routes) <= 0:
+            continue
+        change_routes(action, ptf_ip, port, routes)
 
 
 def get_new_ip(curr_ip, skip_count):
@@ -1474,7 +1561,8 @@ def main():
             adhoc=dict(required=False, type='bool', default=False),
             peers_routes_to_change=dict(required=False, type='dict', default={}),
             log_path=dict(required=False, type='str', default=''),
-            upstream_neighbor_groups=dict(required=False, type='int', default=0)
+            upstream_neighbor_groups=dict(required=False, type='int', default=0),
+            downstream_neighbor_groups=dict(required=False, type='int', default=0)
         ),
         supports_check_mode=False)
 
@@ -1489,6 +1577,7 @@ def main():
     adhoc = module.params['adhoc']
     peers_routes_to_change = module.params['peers_routes_to_change']
     upstream_neighbor_groups = module.params['upstream_neighbor_groups']
+    downstream_neighbor_groups = module.params['downstream_neighbor_groups']
 
     topo = read_topo(topo_name, path)
     if not topo:
@@ -1514,7 +1603,8 @@ def main():
             module.exit_json(changed=True)
         elif topo_type == "t1" or topo_type == "smartswitch-t1":
             fib_t1_lag(
-                topo, ptf_ip, no_default_route=is_storage_backend, action=action, tor_default_route=tor_default_route)
+                topo, ptf_ip, topo_name, no_default_route=is_storage_backend, action=action,
+                tor_default_route=tor_default_route, downstream_neighbor_groups=downstream_neighbor_groups)
             module.exit_json(changed=True)
         elif topo_type == "t2":
             fib_t2_lag(topo, ptf_ip, action=action)

ansible/roles/vm_set/tasks/announce_routes.yml

@@ -126,6 +126,7 @@
       topo_name: "{{ topo }}"
       ptf_ip: "{{ ptf_host_ip }}"
       dut_interfaces: "{{ dut_interfaces | default('') }}"
-      upstream_neighbor_groups: "{{ upstream_neighbor_groups | int }}"
+      upstream_neighbor_groups: "{{ upstream_neighbor_groups | default(0) | int }}"
+      downstream_neighbor_groups: "{{ downstream_neighbor_groups | default(0) | int }}"
     delegate_to: localhost
   when: exabgp_action == 'start'

ansible/testbed-cli.sh

@@ -142,7 +142,7 @@ function read_yaml
 
   tb_line=${tb_lines[0]}
   line_arr=($1)
-  for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 ptf_extra_mgmt_ip netns_mgmt_ip server vm_base dut inv_name auto_recover comment servers upstream_neighbor_groups;
+  for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 ptf_extra_mgmt_ip netns_mgmt_ip server vm_base dut inv_name auto_recover comment servers upstream_neighbor_groups downstream_neighbor_groups;
   do
     value=$(python -c "from __future__ import print_function; tb=eval(\"$tb_line\"); print(tb.get('$attr', None))")
     [ "$value" == "None" ] && value=
@@ -167,6 +167,7 @@ function read_yaml
   inv_name=${line_arr[12]}
   servers=${line_arr[15]}
   upstream_neighbor_groups=${line_arr[16]}
+  downstream_neighbor_groups=${line_arr[17]}
   # Remove the dpu duts by the keyword 'dpu' in the dut name
   duts=$(echo $duts | sed "s/,[^,]*dpu[^,]*//g")
 }
@@ -320,7 +321,7 @@ function add_topo
           -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$vm_set_name" \
           -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" -e ptf_ipv6="$ptf_ipv6" \
           -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" -e netns_mgmt_ip="$netns_mgmt_ip" \
-          -e upstream_neighbor_groups="$upstream_neighbor_groups" \
+          -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \
           $ansible_options $@
 
     if [ $i -eq 0 ]; then
@@ -442,7 +443,7 @@ function renumber_topo
   ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_renumber_vm_topology.yml --vault-password-file="${passwd}" \
       -l "$server" -e testbed_name="$testbed_name" -e duts_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" \
       -e topo="$topo" -e vm_set_name="$vm_set_name" -e ptf_imagename="$ptf_imagename" -e ptf_ipv6="$ptf_ipv6" \
-      -e upstream_neighbor_groups="$upstream_neighbor_groups" \
+      -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \
       -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" $@
 
   ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="${passwd}" -e "dut=$duts" $@
@@ -492,7 +493,7 @@ function refresh_dut
         -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$vm_set_name" \
         -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" -e ptf_ipv6="$ptf_ipv6" \
         -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" -e force_stop_sonic_vm="yes" \
-        -e upstream_neighbor_groups="$upstream_neighbor_groups" \
+        -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \
         $ansible_options $@
 
   echo Done

ansible/vars/topo_t1-isolated-d254u2.yml

@@ -1038,6 +1038,7 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 256
   spine:
     swrole: spine
   tor:

ansible/vars/topo_t1-isolated-d254u2s1.yml

@@ -1042,6 +1042,7 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 256
   spine:
     swrole: spine
   tor:

ansible/vars/topo_t1-isolated-d254u2s2.yml

@@ -1046,6 +1046,7 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 256
   spine:
     swrole: spine
   tor:

ansible/vars/topo_t1-isolated-d510u2.yml

@@ -2062,6 +2062,8 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 512
+    downstream_neighbor_groups: 2
   spine:
     swrole: spine
   tor:

ansible/vars/topo_t1-isolated-d510u2s2.yml

@@ -2070,6 +2070,8 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 256
+    downstream_neighbor_groups: 2
   spine:
     swrole: spine
   tor:

tests/bgp/test_ipv6_bgp_scale.py

@@ -19,7 +19,8 @@
 pytestmark = [
     pytest.mark.topology(
         't0-isolated-d2u254s1', 't0-isolated-d2u254s2', 't0-isolated-d2u510',
-        't1-isolated-d254u2s1', 't1-isolated-d254u2s2', 't1-isolated-d510u2'
+        't1-isolated-d254u2s1', 't1-isolated-d254u2s2', 't1-isolated-d510u2',
+        't1-isolated-d254u2', 't1-isolated-d510u2s2'
     )
 ]
 
@@ -131,7 +132,8 @@ def announce_routes(localhost, tbinfo, ptf_ip, dut_interfaces):
         path="../ansible/",
         log_path="logs",
         dut_interfaces=dut_interfaces,
-        upstream_neighbor_groups=tbinfo['upstream_neighbor_groups'] if 'upstream_neighbor_groups' in tbinfo else None
+        upstream_neighbor_groups=tbinfo['upstream_neighbor_groups'] if 'upstream_neighbor_groups' in tbinfo else 0,
+        downstream_neighbor_groups=tbinfo['downstream_neighbor_groups'] if 'downstream_neighbor_groups' in tbinfo else 0
     )
 
 
@@ -144,7 +146,8 @@ def get_all_bgp_ipv6_routes(duthost):
 def generate_packets(prefixes, dut_mac, src_mac):
     pkts = []
     for prefix in prefixes:
-        addr = str(ipaddress.ip_network(prefix)[1])
+        network = ipaddress.ip_network(prefix)
+        addr = str(network[0] if network.num_addresses == 1 else network[1])
         pkt = simple_icmpv6_packet(
             eth_dst=dut_mac,
             eth_src=src_mac,
@@ -514,7 +517,8 @@ def test_device_unisolation(
     ptfadapter,
     bgp_peers_info,
     setup_packet_mask_counters,
-    announce_bgp_routes_teardown
+    announce_bgp_routes_teardown,
+    tbinfo
 ):
     '''
     This test is for the worst senario that all ports are flapped,