Add static route check for test_nhop_group test (sonic-net#14475)

Add the static route validation in cases of weak switch

Change-Id: I83f2a9a2318adbf901b94f82dcc39b9362d43f78

Author: echuawu

tests/ipfwd/test_nhop_group.py

@@ -325,6 +325,17 @@ def build_pkt(dest_mac, ip_addr, ttl, flow_count):
     return pkt, exp_packet
 
 
+def validate_asic_route(duthost, route, exist=True):
+    logger.info(f"Checking ip route: {route}")
+    asic_info = duthost.shell(f'redis-cli -n 1 keys "ASIC_STATE:SAI_OBJECT_TYPE_ROUTE_ENTRY:*{route}*"',
+                              module_ignore_errors=True)["stdout"]
+    if route in asic_info:
+        logger.info(f"Matched ASIC route: {asic_info}")
+        return exist is True
+    else:
+        return exist is False
+
+
 def test_nhop_group_member_count(duthost, tbinfo, loganalyzer):
     """
     Test next hop group resource count. Steps:
@@ -535,8 +546,9 @@ def built_and_send_tcp_ip_packet():
             nhop.add_ip_route(ip_prefix, ips)
 
             nhop.program_routes()
-            # wait for routes to be synced and programmed
-            time.sleep(15)
+
+            pytest_assert(wait_until(60, 5, 0, validate_asic_route, duthost, ip_prefix),
+                          f"Static route: {ip_prefix} is failed to be programmed!")
 
             ptfadapter.dataplane.flush()
 
@@ -566,6 +578,8 @@ def built_and_send_tcp_ip_packet():
             asic.start_service("bgp")
             time.sleep(15)
             nhop.delete_routes()
+            pytest_assert(wait_until(60, 5, 0, validate_asic_route, duthost, ip_prefix, False),
+                          f"Static route: {ip_prefix} is failed to be removed!")
             arplist.clean_up()
 
     th_asic_flow_map = {0: 'c0:ff:ee:00:00:12', 1: 'c0:ff:ee:00:00:10',