[dualtor] Fix test_orchagent_slb (#18666)

Signed-off-by: Longxiang Lyu <[email protected]>

Author: lolyu

tests/dualtor/test_orchagent_slb.py

@@ -22,7 +22,9 @@
 from tests.common.fixtures.ptfhost_utils import change_mac_addresses                                # noqa F401
 from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory                             # noqa F401
 from tests.common.helpers import bgp
+from tests.common.helpers.assertions import pytest_assert
 from tests.common.utilities import is_ipv4_address
+from tests.common.utilities import wait_until
 
 
 pytestmark = [
@@ -180,7 +182,8 @@ def bgp_neighbors(ptfhost, setup_interfaces):
 def save_slb_exabgp_logfiles(ptfhost, pytestconfig, request):
     """Save slb exabgp log files to the log directory."""
     # remove log files before test
-    log_files_before = ptfhost.shell("ls /tmp/exabgp-slb_*.log")["stdout"].split()
+    log_files_before = ptfhost.shell("ls /tmp/exabgp-slb_*.log*",
+                                     module_ignore_errors=True)["stdout"].split()
     for log_file in log_files_before:
         ptfhost.file(path=log_file, state="absent")
 
@@ -189,7 +192,8 @@ def save_slb_exabgp_logfiles(ptfhost, pytestconfig, request):
     test_log_file = pytestconfig.getoption("log_file", None)
     if test_log_file:
         log_dir = os.path.dirname(os.path.abspath(test_log_file))
-        log_files = ptfhost.shell("ls /tmp/exabgp-slb_*.log")["stdout"].split()
+        log_files = ptfhost.shell("ls /tmp/exabgp-slb_*.log*",
+                                  module_ignore_errors=True)["stdout"].split()
         for log_file in log_files:
             logging.debug("Save slb exabgp log %s to %s", log_file, log_dir)
             ptfhost.fetch(src=log_file, dest=log_dir + os.path.sep, fail_on_missing=False, flat=True)
@@ -256,9 +260,9 @@ def verify_route(duthost, route, existing=True):
         prefix = ipaddress.ip_network(route["prefix"])
         existing_route = duthost.get_ip_route_info(dstip=prefix)
         if existing:
-            assert route["nexthop"] in [str(_[0]) for _ in existing_route["nexthops"]]
+            return route["nexthop"] in [str(_[0]) for _ in existing_route["nexthops"]]
         else:
-            assert len(existing_route["nexthops"]) == 0
+            return len(existing_route["nexthops"]) == 0
 
     def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_existed=True):
 
@@ -311,8 +315,10 @@ def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_
 
         time.sleep(constants.bgp_update_sleep_interval)
 
-        verify_route(upper_tor_host, constants.route, existing=True)
-        verify_route(lower_tor_host, constants.route, existing=True)
+        pytest_assert(verify_route(upper_tor_host, constants.route, existing=True),
+                      "route is not present on the upper ToR")
+        pytest_assert(verify_route(lower_tor_host, constants.route, existing=True),
+                      "route is not present on the lower ToR")
 
         # STEP 3: verify the route by sending some downstream traffic
         verify_traffic(
@@ -330,8 +336,12 @@ def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_
 
         time.sleep(constants.bgp_update_sleep_interval)
 
-        verify_route(upper_tor_host, constants.route, existing=False)
-        verify_route(lower_tor_host, constants.route, existing=False)
+        pytest_assert(wait_until(10, 5, 0, verify_route, upper_tor_host,
+                                 constants.route, existing=False),
+                      "route is not withdrawed from the upper ToR")
+        pytest_assert(wait_until(10, 5, 0, verify_route, lower_tor_host,
+                                 constants.route, existing=False),
+                      "route is not withdrawed from the lower ToR")
 
         # STEP 5: verify the route is removed by verifying that downstream traffic is dropped
         verify_traffic(
@@ -357,8 +367,10 @@ def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_
 
         time.sleep(constants.bgp_update_sleep_interval)
 
-        verify_route(upper_tor_host, constants.route, existing=True)
-        verify_route(lower_tor_host, constants.route, existing=True)
+        pytest_assert(verify_route(upper_tor_host, constants.route, existing=True),
+                      "route is not present on the upper ToR")
+        pytest_assert(verify_route(lower_tor_host, constants.route, existing=True),
+                      "route is not present on the lower ToR")
 
         # STEP 8: verify the route by sending some downstream traffic
         verify_traffic(
@@ -374,7 +386,8 @@ def verify_traffic(duthost, connection, route, is_duthost_active=True, is_route_
         upper_tor_bgp_neighbor.stop_session()
 
         verify_bgp_session(lower_tor_host, lower_tor_bgp_neighbor)
-        verify_route(lower_tor_host, constants.route, existing=True)
+        pytest_assert(verify_route(lower_tor_host, constants.route, existing=True),
+                      "route is not present on the lower ToR")
 
         lower_tor_bgp_neighbor.stop_session()