[action] [PR:19117] [bgp_scale] Fix announce_routes for t1-isolated topo

ansible/library/announce_routes.py

@@ -75,6 +75,10 @@
 IPV6_ADDRESS_PATTERN_DEFAULT_VALUE = '20%02X:%02X%02X:0:%02X::/64'
 ENABLE_IPV4_ROUTES_GENERATION_DEFAULT_VALUE = True
 ENABLE_IPV6_ROUTES_GENERATION_DEFAULT_VALUE = True
+BGP_SCALE_T1S = [
+    't1-isolated-d254u2', 't1-isolated-d254u2s1', 't1-isolated-d254u2s2',
+    't1-isolated-d510u2', 't1-isolated-d510u2s2'
+]
 
 # Describe default number of COLOs
 COLO_NUMBER = 30
@@ -100,6 +104,8 @@
 M0_SUBNET_PREFIX_LEN_V6 = 64
 # Describe default start asn of M1s
 M1_ASN_START = 65200
+# Describe default leaf number
+LEAF_NUMBER = 256
 
 
 def wait_for_http(host_ip, http_port, timeout=10):
@@ -461,6 +467,30 @@ def generate_routes(family, podset_number, tor_number, tor_subnet_number,
     return routes, suffix
 
 
+def generate_t1_to_t0_routes(family, offset, leaf_number, subnet_size, tor_asn, leaf_asn_start, nexthop, nexthop_v6,
+                             podset_num=1, ipv6_address_pattern=IPV6_ADDRESS_PATTERN_DEFAULT_VALUE):
+    routes = []
+    for podset in range(0, podset_num):
+        for leaf in range(0, leaf_number):
+            suffix = offset + leaf
+            octet2 = (168 + int(suffix / (256 ** 2)))
+            octet1 = (192 + int(octet2 / 256))
+            octet2 = (octet2 % 256)
+            octet3 = (int(suffix / 256) % 256)
+            octet4 = (suffix % 256)
+            prefixlen_v4 = (32 - int(math.log(subnet_size, 2)))
+            prefix = "{}.{}.{}.{}/{}".format(octet1, octet2, octet3, octet4, prefixlen_v4)
+            prefix_v6 = ipv6_address_pattern % (
+                octet1, octet2, octet3, octet4)
+            leaf_asn = leaf_asn_start + podset
+            aspath = "{} {}".format(leaf_asn, tor_asn)
+            if family in ["v4", "both"]:
+                routes.append((prefix, nexthop, aspath))
+            if family in ["v6", "both"]:
+                routes.append((prefix_v6, nexthop_v6, aspath))
+    return routes, suffix
+
+
 def fib_t0(topo, ptf_ip, no_default_route=False, action="announce", upstream_neighbor_groups=0):
     common_config = topo['configuration_properties'].get('common', {})
     podset_number = common_config.get("podset_number", PODSET_NUMBER)
@@ -527,7 +557,8 @@ def fib_t0(topo, ptf_ip, no_default_route=False, action="announce", upstream_nei
             current_routes_offset += last_suffix
 
 
-def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_default_route=False):
+def fib_t1_lag(topo, ptf_ip, topo_name, no_default_route=False, action="announce", tor_default_route=False,
+               downstream_neighbor_groups=0):
     common_config = topo['configuration_properties'].get('common', {})
     podset_number = common_config.get("podset_number", PODSET_NUMBER)
     tor_number = common_config.get("tor_number", TOR_NUMBER)
@@ -553,13 +584,22 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_defa
     if 'DPUs' in topo['topology']:
         dpus = topo['topology']['DPUs']
 
+    last_suffix = 0
+    if topo_name in BGP_SCALE_T1S:
+        tor_default_route = True
+    routes_to_change = {}
     for k, v in vms_config.items():
+        curr_no_default_route = no_default_route
+        if topo_name in BGP_SCALE_T1S and 'spine' in v['properties']:
+            curr_no_default_route = True
         if dpus and k in dpus:
             continue
 
         vm_offset = vms[k]['vm_offset']
         port = IPV4_BASE_PORT + vm_offset
         port6 = IPV6_BASE_PORT + vm_offset
+        routes_to_change[port] = []
+        routes_to_change[port6] = []
         aggregate_prefixes = v.get("aggregate_routes", AGGREGATE_ROUTES_DEFAULT_VALUE)
         aggregate_routes = [(prefix, nhipv4 if "." in prefix else nhipv6, "") for prefix in aggregate_prefixes]
         aggregate_routes_v4 = get_ipv4_routes(aggregate_routes)
@@ -574,33 +614,80 @@ def fib_t1_lag(topo, ptf_ip, no_default_route=False, action="announce", tor_defa
         tor_index = tornum - 1 if tornum is not None else None
         if router_type:
             if enable_ipv4_routes_generation:
-                routes_v4, _ = generate_routes("v4", podset_number, tor_number, tor_subnet_number,
-                                               None, leaf_asn_start, tor_asn_start,
-                                               nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
-                                               router_type=router_type, tor_index=tor_index,
-                                               no_default_route=no_default_route, tor_default_route=tor_default_route)
+                routes_v4, last_suffix = generate_routes("v4", podset_number, tor_number, tor_subnet_number,
+                                                         None, leaf_asn_start, tor_asn_start,
+                                                         nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
+                                                         router_type=router_type, tor_index=tor_index,
+                                                         no_default_route=curr_no_default_route,
+                                                         tor_default_route=tor_default_route)
                 if aggregate_routes_v4:
                     filterout_subnet_ipv4(aggregate_routes, routes_v4)
                     routes_v4.extend(aggregate_routes_v4)
-                change_routes(action, ptf_ip, port, routes_v4)
+                routes_to_change[port] += routes_v4
             if enable_ipv6_routes_generation:
-                routes_v6, _ = generate_routes("v6", podset_number, tor_number, tor_subnet_number,
-                                               None, leaf_asn_start, tor_asn_start,
-                                               nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
-                                               router_type=router_type, tor_index=tor_index,
-                                               no_default_route=no_default_route,
-                                               ipv6_address_pattern=ipv6_address_pattern,
-                                               tor_default_route=tor_default_route)
+                routes_v6, last_suffix = generate_routes("v6", podset_number, tor_number, tor_subnet_number,
+                                                         None, leaf_asn_start, tor_asn_start,
+                                                         nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
+                                                         router_type=router_type, tor_index=tor_index,
+                                                         no_default_route=curr_no_default_route,
+                                                         ipv6_address_pattern=ipv6_address_pattern,
+                                                         tor_default_route=tor_default_route)
                 if aggregate_routes_v6:
                     filterout_subnet_ipv6(aggregate_routes, routes_v6)
                     routes_v6.extend(aggregate_routes_v6)
-                change_routes(action, ptf_ip, port6, routes_v6)
+                routes_to_change[port6] += routes_v6
 
         if 'vips' in v:
             routes_vips = []
             for prefix in v["vips"]["ipv4"]["prefixes"]:
                 routes_vips.append((prefix, nhipv4, v["vips"]["ipv4"]["asn"]))
-            change_routes(action, ptf_ip, port, routes_vips)
+            routes_to_change[port] += routes_vips
+    if topo_name in BGP_SCALE_T1S:
+        if downstream_neighbor_groups == 0:
+            downstream_neighbor_groups = common_config.get("downstream_neighbor_groups", DEFAULT_NEIGHBOR_GROUPS)
+
+        # Announce T1 loopback received in T0
+        leaf_number = common_config.get("leaf_number", LEAF_NUMBER)
+        tor_number = len([k for k, v in vms_config.items() if 'tor' in v['properties']])
+        lov6_address_pattern = ipv6_address_pattern.split("/")[0] + "/128"
+        current_routes_offset = last_suffix
+        for index, (k, v) in enumerate(vms_config.items()):
+            if dpus and k in dpus:
+                continue
+            vm_offset = vms[k]['vm_offset']
+            port = IPV4_BASE_PORT + vm_offset
+            port6 = IPV6_BASE_PORT + vm_offset
+            aggregate_prefixes = v.get("aggregate_routes", AGGREGATE_ROUTES_DEFAULT_VALUE)
+            aggregate_routes = [(prefix, nhipv4 if "." in prefix else nhipv6, "") for prefix in aggregate_prefixes]
+            aggregate_routes_v4 = get_ipv4_routes(aggregate_routes)
+            aggregate_routes_v6 = get_ipv6_routes(aggregate_routes)
+            if 'spine' in v['properties']:
+                continue
+            tor_asn = tor_asn_start + index
+            if enable_ipv4_routes_generation:
+                routes_v4, last_suffix = generate_t1_to_t0_routes("v4", current_routes_offset, leaf_number, 1, tor_asn,
+                                                                  leaf_asn_start, nhipv4, nhipv6,
+                                                                  ipv6_address_pattern=lov6_address_pattern)
+                if aggregate_routes_v4:
+                    filterout_subnet_ipv4(aggregate_routes, routes_v4)
+                    routes_v4.extend(aggregate_routes_v4)
+                routes_to_change[port] += routes_v4
+            if enable_ipv6_routes_generation:
+                routes_v6, last_suffix = generate_t1_to_t0_routes("v6", current_routes_offset, leaf_number, 1, tor_asn,
+                                                                  leaf_asn_start, nhipv6, nhipv6,
+                                                                  ipv6_address_pattern=lov6_address_pattern)
+                if aggregate_routes_v6:
+                    filterout_subnet_ipv6(aggregate_routes, routes_v6)
+                    routes_v6.extend(aggregate_routes_v6)
+                routes_to_change[port6] += routes_v6
+            group_index = index * downstream_neighbor_groups // tor_number
+            next_group_index = (index + 1) * downstream_neighbor_groups // tor_number
+            if group_index != next_group_index:
+                current_routes_offset += last_suffix
+    for port, routes in routes_to_change.items():
+        if len(routes) <= 0:
+            continue
+        change_routes(action, ptf_ip, port, routes)
 
 
 def get_new_ip(curr_ip, skip_count):
@@ -1194,7 +1281,8 @@ def main():
             adhoc=dict(required=False, type='bool', default=False),
             peers_routes_to_change=dict(required=False, type='dict', default={}),
             log_path=dict(required=False, type='str', default=''),
-            upstream_neighbor_groups=dict(required=False, type='int', default=0)
+            upstream_neighbor_groups=dict(required=False, type='int', default=0),
+            downstream_neighbor_groups=dict(required=False, type='int', default=0)
         ),
         supports_check_mode=False)
 
@@ -1209,6 +1297,7 @@ def main():
     adhoc = module.params['adhoc']
     peers_routes_to_change = module.params['peers_routes_to_change']
     upstream_neighbor_groups = module.params['upstream_neighbor_groups']
+    downstream_neighbor_groups = module.params['downstream_neighbor_groups']
 
     topo = read_topo(topo_name, path)
     if not topo:
@@ -1234,7 +1323,8 @@ def main():
             module.exit_json(changed=True)
         elif topo_type == "t1" or topo_type == "smartswitch-t1":
             fib_t1_lag(
-                topo, ptf_ip, no_default_route=is_storage_backend, action=action, tor_default_route=tor_default_route)
+                topo, ptf_ip, topo_name, no_default_route=is_storage_backend, action=action,
+                tor_default_route=tor_default_route, downstream_neighbor_groups=downstream_neighbor_groups)
             module.exit_json(changed=True)
         elif topo_type == "t2":
             fib_t2_lag(topo, ptf_ip, action=action)

ansible/roles/vm_set/tasks/announce_routes.yml

@@ -127,6 +127,7 @@
       topo_name: "{{ topo }}"
       ptf_ip: "{{ ptf_host_ip }}"
       dut_interfaces: "{{ dut_interfaces | default('') }}"
-      upstream_neighbor_groups: "{{ upstream_neighbor_groups | int }}"
+      upstream_neighbor_groups: "{{ upstream_neighbor_groups | default(0) | int }}"
+      downstream_neighbor_groups: "{{ downstream_neighbor_groups | default(0) | int }}"
     delegate_to: localhost
   when: exabgp_action == 'start'

ansible/testbed-cli.sh

@@ -142,7 +142,7 @@ function read_yaml
 
   tb_line=${tb_lines[0]}
   line_arr=($1)
-  for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 ptf_extra_mgmt_ip netns_mgmt_ip server vm_base dut inv_name auto_recover comment servers upstream_neighbor_groups;
+  for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 ptf_extra_mgmt_ip netns_mgmt_ip server vm_base dut inv_name auto_recover comment servers upstream_neighbor_groups downstream_neighbor_groups;
   do
     value=$(python -c "from __future__ import print_function; tb=eval(\"$tb_line\"); print(tb.get('$attr', None))")
     [ "$value" == "None" ] && value=
@@ -167,6 +167,7 @@ function read_yaml
   inv_name=${line_arr[12]}
   servers=${line_arr[15]}
   upstream_neighbor_groups=${line_arr[16]}
+  downstream_neighbor_groups=${line_arr[17]}
   # Remove the dpu duts by the keyword 'dpu' in the dut name
   duts=$(echo $duts | sed "s/,[^,]*dpu[^,]*//g")
 }
@@ -320,7 +321,7 @@ function add_topo
           -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$vm_set_name" \
           -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" -e ptf_ipv6="$ptf_ipv6" \
           -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" -e netns_mgmt_ip="$netns_mgmt_ip" \
-          -e upstream_neighbor_groups="$upstream_neighbor_groups" \
+          -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \
           $ansible_options $@
 
     if [ $i -eq 0 ]; then
@@ -438,7 +439,7 @@ function renumber_topo
   ANSIBLE_SCP_IF_SSH=y ansible-playbook -i $vmfile testbed_renumber_vm_topology.yml --vault-password-file="${passwd}" \
       -l "$server" -e testbed_name="$testbed_name" -e duts_name="$duts" -e VM_base="$vm_base" -e ptf_ip="$ptf_ip" \
       -e topo="$topo" -e vm_set_name="$vm_set_name" -e ptf_imagename="$ptf_imagename" -e ptf_ipv6="$ptf_ipv6" \
-      -e upstream_neighbor_groups="$upstream_neighbor_groups" \
+      -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \
       -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" $@
 
   ansible-playbook fanout_connect.yml -i $vmfile --limit "$server" --vault-password-file="${passwd}" -e "dut=$duts" $@
@@ -488,7 +489,7 @@ function refresh_dut
         -e ptf_ip="$ptf_ip" -e topo="$topo" -e vm_set_name="$vm_set_name" \
         -e ptf_imagename="$ptf_imagename" -e vm_type="$vm_type" -e ptf_ipv6="$ptf_ipv6" \
         -e ptf_extra_mgmt_ip="$ptf_extra_mgmt_ip" -e force_stop_sonic_vm="yes" \
-        -e upstream_neighbor_groups="$upstream_neighbor_groups" \
+        -e upstream_neighbor_groups="$upstream_neighbor_groups" -e downstream_neighbor_groups="$downstream_neighbor_groups" \
         $ansible_options $@
 
   echo Done

ansible/vars/topo_t1-isolated-d254u2.yml

@@ -1038,6 +1038,7 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 256
   spine:
     swrole: spine
   tor:

ansible/vars/topo_t1-isolated-d254u2s1.yml

@@ -1042,6 +1042,7 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 256
   spine:
     swrole: spine
   tor:

ansible/vars/topo_t1-isolated-d254u2s2.yml

@@ -1046,6 +1046,7 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 256
   spine:
     swrole: spine
   tor:

ansible/vars/topo_t1-isolated-d510u2.yml

@@ -2062,6 +2062,8 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 512
+    downstream_neighbor_groups: 2
   spine:
     swrole: spine
   tor:

ansible/vars/topo_t1-isolated-d510u2s2.yml

@@ -2070,6 +2070,8 @@ configuration_properties:
     ipv6_address_pattern: FC00:C:C::%02X%02X:%02X%02X:0/120
     enable_ipv4_routes_generation: false
     enable_ipv6_routes_generation: true
+    leaf_number: 256
+    downstream_neighbor_groups: 2
   spine:
     swrole: spine
   tor:

tests/bgp/test_ipv6_bgp_scale.py

@@ -19,7 +19,8 @@
 pytestmark = [
     pytest.mark.topology(
         't0-isolated-d2u254s1', 't0-isolated-d2u254s2', 't0-isolated-d2u510',
-        't1-isolated-d254u2s1', 't1-isolated-d254u2s2', 't1-isolated-d510u2'
+        't1-isolated-d254u2s1', 't1-isolated-d254u2s2', 't1-isolated-d510u2',
+        't1-isolated-d254u2', 't1-isolated-d510u2s2'
     )
 ]
 
@@ -131,7 +132,8 @@ def announce_routes(localhost, tbinfo, ptf_ip, dut_interfaces):
         path="../ansible/",
         log_path="logs",
         dut_interfaces=dut_interfaces,
-        upstream_neighbor_groups=tbinfo['upstream_neighbor_groups'] if 'upstream_neighbor_groups' in tbinfo else None
+        upstream_neighbor_groups=tbinfo['upstream_neighbor_groups'] if 'upstream_neighbor_groups' in tbinfo else 0,
+        downstream_neighbor_groups=tbinfo['downstream_neighbor_groups'] if 'downstream_neighbor_groups' in tbinfo else 0
     )
 
 
@@ -144,7 +146,8 @@ def get_all_bgp_ipv6_routes(duthost):
 def generate_packets(prefixes, dut_mac, src_mac):
     pkts = []
     for prefix in prefixes:
-        addr = str(ipaddress.ip_network(prefix)[1])
+        network = ipaddress.ip_network(prefix)
+        addr = str(network[0] if network.num_addresses == 1 else network[1])
         pkt = simple_icmpv6_packet(
             eth_dst=dut_mac,
             eth_src=src_mac,
@@ -514,7 +517,8 @@ def test_device_unisolation(
     ptfadapter,
     bgp_peers_info,
     setup_packet_mask_counters,
-    announce_bgp_routes_teardown
+    announce_bgp_routes_teardown,
+    tbinfo
 ):
     '''
     This test is for the worst senario that all ports are flapped,