Fix/nonlinear high nexthop dataplane downtime (sonic-net#21939) (sonic-net#22204)

* ptf dataplane cleaners for in between test runs

Signed-off-by: Priyansh Tratiya <[email protected]>
Co-authored-by: Priyansh <[email protected]>
Signed-off-by: Lakshmi Yarramaneni <[email protected]>

Author: 2 people

tests/bgp/test_ipv6_bgp_scale.py

@@ -620,8 +620,8 @@ def test_nexthop_group_member_scale(
     localhost,
     tbinfo,
     bgp_peers_info,
+    clean_ptf_dataplane,
     setup_routes_before_test,
-    topo_bgp_routes,
     request
 ):
     '''
@@ -645,6 +645,7 @@ def test_nexthop_group_member_scale(
     global global_icmp_type
     global_icmp_type += 1
     pdp = ptfadapter.dataplane
+    pdp.clear_masks()
     pdp.set_qlen(PACKET_QUEUE_LENGTH)
     exp_mask = setup_packet_mask_counters(pdp, global_icmp_type)
     injection_bgp_neighbor = random.choice(list(bgp_peers_info.keys()))
@@ -716,34 +717,28 @@ def test_nexthop_group_member_scale(
         terminated.set()
         traffic_thread.join()
         end_time = datetime.datetime.now()
-        validate_rx_tx_counters(pdp, end_time, start_time, exp_mask, _get_max_time('dataplane_downtime', 1))
+        acceptable_downtime = validate_rx_tx_counters(pdp, end_time, start_time, exp_mask,
+                                                      _get_max_time('dataplane_downtime', 1))
+        if not acceptable_downtime:
+            for ptfhost in ptfhosts:
+                ptf_ip = ptfhost.mgmt_ip
+                announce_routes(localhost, tbinfo, ptf_ip, servers_dut_interfaces.get(ptf_ip, ''))
+            pytest.fail(f"Dataplane downtime is too high, threshold is "
+                        f"{_get_max_time('dataplane_downtime', 1)} seconds")
         if not result.get("converged"):
             pytest.fail("BGP routes are not stable in long time")
     finally:
-        for ptfhost in ptfhosts:
-            ptf_ip = ptfhost.mgmt_ip
-            change_routes_on_peers(localhost, ptf_ip, topo_name, peers_routes_to_change, ACTION_ANNOUNCE,
-                                   servers_dut_interfaces.get(ptf_ip, ''))
+        pass
     # ------------announce routes and test ------------ #
     current_test = request.node.name + "_announce"
     global_icmp_type += 1
+    pdp.clear_masks()
     exp_mask = setup_packet_mask_counters(pdp, global_icmp_type)
     pkts = generate_packets(
         neighbor_ecmp_routes[injection_bgp_neighbor],
         duthost.facts['router_mac'],
         pdp.get_mac(pdp.port_to_device(injection_port), injection_port)
     )
-    for hostname, routes in peers_routes_to_change.items():
-        for route in routes:
-            prefix = route[0].upper()
-            found = False
-            for topo_route in topo_bgp_routes[hostname]['ipv6']:
-                if topo_route[0] == prefix:
-                    route[2] = topo_route[2]
-                    found = True
-                    break
-            if not found:
-                logger.warning('Fail to update AS path of route %s, because of prefix was not found in topo', route[0])
     terminated = Event()
     traffic_thread = Thread(
         target=send_packets, args=(terminated, pdp, pdp.port_to_device(injection_port), injection_port, pkts)
@@ -753,8 +748,7 @@ def test_nexthop_group_member_scale(
     traffic_thread.start()
     for ptfhost in ptfhosts:
         ptf_ip = ptfhost.mgmt_ip
-        change_routes_on_peers(localhost, ptf_ip, topo_name, peers_routes_to_change, ACTION_ANNOUNCE,
-                               servers_dut_interfaces.get(ptf_ip, ''))
+        announce_routes(localhost, tbinfo, ptf_ip, servers_dut_interfaces.get(ptf_ip, ''))
     compressed_startup_routes = compress_expected_routes(startup_routes)
     result = check_bgp_routes_converged(
         duthost=duthost,
@@ -769,7 +763,10 @@ def test_nexthop_group_member_scale(
     terminated.set()
     traffic_thread.join()
     end_time = datetime.datetime.now()
-    validate_rx_tx_counters(pdp, end_time, start_time, exp_mask, _get_max_time('dataplane_downtime', 1))
+    acceptable_downtime = validate_rx_tx_counters(pdp, end_time, start_time, exp_mask,
+                                                  _get_max_time('dataplane_downtime', 1))
+    if not acceptable_downtime:
+        pytest.fail(f"Dataplane downtime is too high, threshold is {_get_max_time('dataplane_downtime', 1)} seconds")
     if not result.get("converged"):
         pytest.fail("BGP routes are not stable in long time")