Improves wait_until to support initial delay before periodic check and wait

tests/acl/test_acl.py

@@ -534,7 +534,7 @@ def direction(self, request):
     def check_rule_counters(self, duthost):
         logger.info('Wait all rule counters are ready')
 
-        return wait_until(60, 2, self.check_rule_counters_internal, duthost)
+        return wait_until(60, 2, 0, self.check_rule_counters_internal, duthost)
 
     def check_rule_counters_internal(self, duthost):
         for asic_id in duthost.get_frontend_asic_ids():

tests/arp/test_neighbor_mac_noptf.py

@@ -71,7 +71,7 @@ def setupDutConfig(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname):
         duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
         if not duthost.get_facts().get("modular_chassis"):
             duthost.command("sudo config bgp shutdown all")
-            if not wait_until(120, 2.0, self._check_no_bgp_routes, duthost):
+            if not wait_until(120, 2.0, 0, self._check_no_bgp_routes, duthost):
                 pytest.fail('BGP Shutdown Timeout: BGP route removal exceeded 120 seconds.')
 
         yield
@@ -115,7 +115,7 @@ def find_routed_interface():
                         testRoutedInterface[asichost.asic_index] = intf
             return testRoutedInterface
 
-        if not wait_until(120, 2, find_routed_interface):
+        if not wait_until(120, 2, 0, find_routed_interface):
             pytest.fail('Failed to find routed interface in 120 s')
 
         yield testRoutedInterface

tests/autorestart/test_container_autorestart.py

@@ -218,6 +218,7 @@ def clear_failed_flag_and_restart(duthost, container_name):
     duthost.shell("sudo systemctl start {}.service".format(container_name))
     restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS,
                            CONTAINER_CHECK_INTERVAL_SECS,
+                           0,
                            check_container_state, duthost, container_name, True)
     pytest_assert(restarted, "Failed to restart container '{}' after reset-failed was cleared".format(container_name))
 
@@ -240,13 +241,15 @@ def verify_autorestart_with_critical_process(duthost, container_name, program_na
     logger.info("Waiting until container '{}' is stopped...".format(container_name))
     stopped = wait_until(CONTAINER_STOP_THRESHOLD_SECS,
                          CONTAINER_CHECK_INTERVAL_SECS,
+                         0,
                          check_container_state, duthost, container_name, False)
     pytest_assert(stopped, "Failed to stop container '{}'".format(container_name))
     logger.info("Container '{}' was stopped".format(container_name))
 
     logger.info("Waiting until container '{}' is restarted...".format(container_name))
     restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS,
                            CONTAINER_CHECK_INTERVAL_SECS,
+                           0,
                            check_container_state, duthost, container_name, True)
     if not restarted:
         if is_hiting_start_limit(duthost, container_name):
@@ -275,6 +278,7 @@ def verify_no_autorestart_with_non_critical_process(duthost, container_name, pro
     logger.info("Waiting to ensure container '{}' does not stop...".format(container_name))
     stopped = wait_until(CONTAINER_STOP_THRESHOLD_SECS,
                          CONTAINER_CHECK_INTERVAL_SECS,
+                         0,
                          check_container_state, duthost, container_name, False)
     pytest_assert(not stopped, "Container '{}' was stopped unexpectedly".format(container_name))
     logger.info("Container '{}' did not stop".format(container_name))
@@ -333,7 +337,7 @@ def postcheck_critical_processes_status(duthost, container_autorestart_states, u
         if is_hiting_start_limit(duthost, container_name):
             clear_failed_flag_and_restart(duthost, container_name)
 
-    return wait_until(POST_CHECK_THRESHOLD_SECS, POST_CHECK_INTERVAL_SECS,
+    return wait_until(POST_CHECK_THRESHOLD_SECS, POST_CHECK_INTERVAL_SECS, 0,
                       post_test_check, duthost, up_bgp_neighbors)
 
 

tests/bgp/bgp_helpers.py

@@ -44,7 +44,7 @@ def restart_bgp(duthost, asic_index=DEFAULT_ASIC_ID):
     duthost.asic_instance(asic_index).reset_service("bgp")
     duthost.asic_instance(asic_index).restart_service("bgp")
     docker_name = duthost.asic_instance(asic_index).get_docker_name("bgp")
-    pytest_assert(wait_until(100, 10, duthost.is_service_fully_started, docker_name), "BGP not started.")
+    pytest_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, docker_name), "BGP not started.")
 
 
 def define_config(duthost, template_src_path, template_dst_path):
@@ -128,7 +128,7 @@ def get_routes_not_announced_to_bgpmon(duthost, ptfhost):
     """
     def _dump_fie_exists(host):
         return host.stat(path=DUMP_FILE).get('stat', {}).get('exists', False)
-    pytest_assert(wait_until(120, 10, _dump_fie_exists, ptfhost))
+    pytest_assert(wait_until(120, 10, 0, _dump_fie_exists, ptfhost))
     time.sleep(20)  # Wait until all routes announced to bgpmon
     bgpmon_routes = parse_exabgp_dump(ptfhost)
     rib_v4 = parse_rib(duthost, 4)

tests/bgp/conftest.py

@@ -137,12 +137,12 @@ def restore_nbr_gr(node=None, results=None):
     logger.info("bgp neighbors: {}".format(bgp_neighbors.keys()))
     res = True
     err_msg = ""
-    if not wait_until(300, 10, duthost.check_bgp_session_state, bgp_neighbors.keys()):
+    if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()):
         res = False
         err_msg = "not all bgp sessions are up after enable graceful restart"
 
     is_backend_topo = "backend" in tbinfo["topo"]["name"]
-    if not is_backend_topo and res and not wait_until(100, 5, duthost.check_bgp_default_route):
+    if not is_backend_topo and res and not wait_until(100, 5, 0, duthost.check_bgp_default_route):
         res = False
         err_msg = "ipv4 or ipv6 bgp default route not available"
 
@@ -157,7 +157,7 @@ def restore_nbr_gr(node=None, results=None):
 
     check_results(results)
 
-    if not wait_until(300, 10, duthost.check_bgp_session_state, bgp_neighbors.keys()):
+    if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()):
         pytest.fail("not all bgp sessions are up after disable graceful restart")
 
 
@@ -530,7 +530,7 @@ def bgpmon_setup_teardown(ptfhost, duthost, localhost, setup_interfaces):
 
     pt_assert(wait_tcp_connection(localhost, ptfhost.mgmt_ip, BGP_MONITOR_PORT),
                   "Failed to start bgp monitor session on PTF")
-    pt_assert(wait_until(20, 5, duthost.check_bgp_session_state, [peer_addr]), 'BGP session {} on duthost is not established'.format(BGP_MONITOR_NAME))
+    pt_assert(wait_until(20, 5, 0, duthost.check_bgp_session_state, [peer_addr]), 'BGP session {} on duthost is not established'.format(BGP_MONITOR_NAME))
 
     yield
     # Cleanup bgp monitor

tests/bgp/test_bgp_bbr.py

@@ -292,10 +292,10 @@ def check_other_vms(nbrhosts, setup, route, accepted=True, node=None, results=No
     bgp_neighbors = json.loads(duthost.shell("sonic-cfggen -d --var-json 'BGP_NEIGHBOR'")['stdout'])
 
     # check tor1
-    pytest_assert(wait_until(5, 1, check_tor1, nbrhosts, setup, route), 'tor1 check failed')
+    pytest_assert(wait_until(5, 1, 0, check_tor1, nbrhosts, setup, route), 'tor1 check failed')
 
     # check DUT
-    pytest_assert(wait_until(5, 1, check_dut, duthost, other_vms, bgp_neighbors, setup, route, accepted=accepted), 'DUT check failed')
+    pytest_assert(wait_until(5, 1, 0, check_dut, duthost, other_vms, bgp_neighbors, setup, route, accepted=accepted), 'DUT check failed')
 
     results = parallel_run(check_other_vms, (nbrhosts, setup, route), {'accepted': accepted}, other_vms, timeout=120)
 

tests/bgp/test_bgp_gr_helper.py

@@ -160,7 +160,7 @@ def _verify_prefix_counters_from_neighbor_after_graceful_restart(duthost, bgp_ne
         # wait till DUT enters NSF state
         for test_bgp_neighbor in test_bgp_neighbors:
             pytest_assert(
-                wait_until(60, 5, duthost.check_bgp_session_nsf, test_bgp_neighbor),
+                wait_until(60, 5, 0, duthost.check_bgp_session_nsf, test_bgp_neighbor),
                 "neighbor {} does not enter NSF state".format(test_bgp_neighbor)
             )
 
@@ -187,7 +187,7 @@ def _verify_prefix_counters_from_neighbor_after_graceful_restart(duthost, bgp_ne
 
         # wait for exabgp sessions to establish
         pytest_assert(
-            wait_until(300, 10, test_neighbor_host.check_bgp_session_state, exabgp_ips, exabgp_sessions),
+            wait_until(300, 10, 0, test_neighbor_host.check_bgp_session_state, exabgp_ips, exabgp_sessions),
             "exabgp sessions {} are not coming back".format(exabgp_sessions)
         )
 
@@ -199,13 +199,13 @@ def _verify_prefix_counters_from_neighbor_after_graceful_restart(duthost, bgp_ne
 
     # confirm BGP session are up
     pytest_assert(
-        wait_until(300, 10, duthost.check_bgp_session_state, test_bgp_neighbors),
+        wait_until(300, 10, 0, duthost.check_bgp_session_state, test_bgp_neighbors),
         "graceful restarted bgp sessions {} are not coming back".format(test_bgp_neighbors)
     )
 
     # confirm routes from the neighbor are restored
     pytest_assert(
-        wait_until(300, 10, _verify_prefix_counters_from_neighbor_after_graceful_restart, duthost, test_bgp_neighbors),
+        wait_until(300, 10, 0, _verify_prefix_counters_from_neighbor_after_graceful_restart, duthost, test_bgp_neighbors),
         "after graceful restart, Rib is not restored"
     )
 

tests/bgp/test_bgpmon.py

@@ -151,7 +151,7 @@ def bgpmon_peer_connected(duthost, bgpmon_peer):
     try:
         pytest_assert(wait_tcp_connection(localhost, ptfhost.mgmt_ip, BGP_MONITOR_PORT),
                       "Failed to start bgp monitor session on PTF")
-        pytest_assert(wait_until(180, 5, bgpmon_peer_connected, duthost, peer_addr),"BGPMon Peer connection not established")
+        pytest_assert(wait_until(180, 5, 0, bgpmon_peer_connected, duthost, peer_addr),"BGPMon Peer connection not established")
     finally:
         ptfhost.exabgp(name=BGP_MONITOR_NAME, state="absent")
         ptfhost.shell("ip route del %s dev %s" % (local_addr + "/32", ptf_interface))

tests/common/dualtor/control_plane_utils.py

@@ -69,7 +69,7 @@ def _dump_db(self, db, key_pattern):
 
     def verify_db(self, db):
         pytest_assert(
-            wait_until(30, 10, self.get_mismatched_ports, db),
+            wait_until(30, 10, 0, self.get_mismatched_ports, db),
             "Database states don't match expected state {state},"
             "incorrect {db_name} values {db_states}"
             .format(state=self.state, db_name=DB_NAME_MAP[db],

tests/common/dualtor/data_plane_utils.py

@@ -153,7 +153,7 @@ def run_test(duthosts, activehost, ptfhost, ptfadapter, action,
         action()
     # do not time-wait the test, if early stop is not requested (when stop_after=None)
     if stop_after is not None:
-        wait_until(timeout=stop_after, interval=0.5, condition=\
+        wait_until(timeout=stop_after, interval=0.5, 0, condition=\
             lambda: not send_and_sniff.is_alive)
         if send_and_sniff.is_alive():
             logger.info("Sender/Sniffer threads are still running. Sending signal "\

tests/common/dualtor/mux_simulator_control.py

@@ -480,7 +480,7 @@ def _check_mux_status_consistency():
     mg_facts = upper_tor_host.get_extended_minigraph_facts(tbinfo)
     port_indices = mg_facts['minigraph_port_indices']
     pytest_assert(
-        utilities.wait_until(30, 5, _check_mux_status_consistency),
+        utilities.wait_until(30, 5, 0, _check_mux_status_consistency),
         "Mux status is inconsistent between the DUTs and mux simulator after toggle"
     )
 

tests/common/dualtor/tunnel_traffic_utils.py

@@ -143,7 +143,7 @@ def _disassemble_ip_tos(tos):
                 check_res.append("outer packet DSCP not same as inner packet DSCP")
             exp_queue = derive_queue_id_from_dscp(outer_dscp)
 
-            pytest_assert(wait_until(60, 5, queue_stats_check, self.standby_tor, exp_queue))
+            pytest_assert(wait_until(60, 5, 0, queue_stats_check, self.standby_tor, exp_queue))
             return check_res
 
         def __init__(self, standby_tor, active_tor=None, existing=True):

tests/common/helpers/drop_counters/drop_counters.py

@@ -94,7 +94,7 @@ def get_drops_across_all_duthosts():
             drop_list.append(int(get_pkt_drops(duthost, get_cnt_cli_cmd, asic_index)[dut_iface][column_key].replace(",", "")))
         return drop_list
     check_drops_on_dut = lambda: packets_count in get_drops_across_all_duthosts()
-    if not wait_until(25, 1, check_drops_on_dut):
+    if not wait_until(25, 1, 0, check_drops_on_dut):
         # The actual Drop count should always be equal or 1 or 2 packets more than what is expected due to some other drop may occur
         # over the interface being examined. When that happens if looking onlyu for exact count it will be a false positive failure.
         # So do one more check to allow up to 2 packets more dropped than what was expected as an allowed case.

tests/common/helpers/dut_utils.py

@@ -88,6 +88,7 @@ def clear_failed_flag_and_restart(duthost, container_name):
     duthost.shell("sudo systemctl start {}.service".format(container_name))
     restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS,
                            CONTAINER_CHECK_INTERVAL_SECS,
+                           0,
                            check_container_state, duthost, container_name, True)
     pytest_assert(restarted, "Failed to restart container '{}' after reset-failed was cleared".format(container_name))
 

tests/common/helpers/snmp_helpers.py

@@ -35,7 +35,7 @@ def get_snmp_facts(localhost, host, version, community, is_dell=False, module_ig
 
     global global_snmp_facts
 
-    pytest_assert(wait_until(timeout, interval, _update_snmp_facts, localhost, host, version,
+    pytest_assert(wait_until(timeout, interval, 0, _update_snmp_facts, localhost, host, version,
                              community, is_dell), "Timeout waiting for SNMP facts")
     return global_snmp_facts
 

tests/common/helpers/voq_lag.py

@@ -91,7 +91,7 @@ def add_lag(duthost, asic, portchannel_members=None, portchannel_ip=None,
         pytest_assert(int_facts['ansible_interface_facts']
                       [portchannel]['ipv4']['address'] == portchannel_ip.split('/')[0])
 
-        pytest_assert(wait_until(30,5, verify_lag_interface, duthost, asic, portchannel),
+        pytest_assert(wait_until(30,5, 0, verify_lag_interface, duthost, asic, portchannel),
                       'For added Portchannel {} link is not up'.format(portchannel))
 
 
@@ -209,7 +209,7 @@ def delete_lag_members_ip(duthost, asic, portchannel_members,
         duthost.shell("config interface {} ip remove {} {}"
                       .format(asic.cli_ns_option, portchannel, portchannel_ip))
 
-        pytest_assert(wait_until(30,5, verify_lag_interface, duthost, asic, portchannel, expected=False),
+        pytest_assert(wait_until(30,5, 0, verify_lag_interface, duthost, asic, portchannel, expected=False),
                       'For deleted Portchannel {} ip link is not down'.format(portchannel))
 
 

tests/common/platform/processes_utils.py

@@ -44,6 +44,6 @@ def wait_critical_processes(dut):
     @param dut: The AnsibleHost object of DUT. For interacting with DUT.
     """
     logging.info("Wait until all critical processes are healthy")
-    pytest_assert(wait_until(300, 20, _all_critical_processes_healthy, dut),
+    pytest_assert(wait_until(300, 20, 0, _all_critical_processes_healthy, dut),
                   "Not all critical processes are healthy")
 

tests/common/plugins/ptfadapter/ptfadapter.py

@@ -59,7 +59,7 @@ def _check_ptf_nn_agent_availability(self, socket_addr):
         sock = nnpy.Socket(nnpy.AF_SP, nnpy.PAIR)
         sock.connect(socket_addr)
         try:
-            return wait_until(1, 0.2, lambda:sock.get_statistic(self.NN_STAT_CURRENT_CONNECTIONS) == 1)
+            return wait_until(1, 0.2, 0, lambda:sock.get_statistic(self.NN_STAT_CURRENT_CONNECTIONS) == 1)
         finally:
             sock.close()
 

tests/common/plugins/sanity_check/checks.py

@@ -238,7 +238,7 @@ def _check_bgp_status_helper():
         networking_uptime = dut.get_networking_uptime().seconds
         timeout = max(SYSTEM_STABILIZE_MAX_TIME - networking_uptime, 1)
         interval = 20
-        wait_until(timeout, interval, _check_bgp_status_helper)
+        wait_until(timeout, interval, 0, _check_bgp_status_helper)
         if (check_result['failed']):
             for a_result in check_result.keys():
                 if a_result != 'failed':

tests/common/port_toggle.py

@@ -60,7 +60,7 @@ def __get_down_ports(expect_up=True):
             log_system_resources(duthost, logger)
 
         logger.info("Wait for ports to go down")
-        shutdown_ok = wait_until(port_down_wait_time, 5, lambda: len(__get_down_ports(expect_up=False)) == len(ports))
+        shutdown_ok = wait_until(port_down_wait_time, 5, 0, lambda: len(__get_down_ports(expect_up=False)) == len(ports))
 
         if not shutdown_ok:
             up_ports = __get_down_ports(expect_up=True)
@@ -74,7 +74,7 @@ def __get_down_ports(expect_up=True):
         duthost.shell_cmds(cmds=cmds_up)
 
         logger.info("Wait for ports to come up")
-        startup_ok = wait_until(port_up_wait_time, 5, lambda: len(__get_down_ports()) == 0)
+        startup_ok = wait_until(port_up_wait_time, 5, 0, lambda: len(__get_down_ports()) == 0)
 
         if not startup_ok:
             down_ports = __get_down_ports()

tests/common/system_utils/docker.py

@@ -188,15 +188,15 @@ def ready_for_swap():
 
         return True
 
-    shutdown_check = wait_until(30, 3, ready_for_swap)
+    shutdown_check = wait_until(30, 3, 0, ready_for_swap)
     pytest_assert(shutdown_check, "Docker and/or BGP failed to shut down in 30s")
 
 
 def _perform_syncd_liveness_check(duthost):
     def check_liveness():
         return duthost.is_service_running("syncd")
 
-    liveness_check = wait_until(30, 1, check_liveness)
+    liveness_check = wait_until(30, 1, 0, check_liveness)
     pytest_assert(liveness_check, "syncd crashed after swap_syncd")
 
 

tests/common/utilities.py

@@ -73,19 +73,25 @@ def wait(seconds, msg=""):
     time.sleep(seconds)
 
 
-def wait_until(timeout, interval, condition, *args, **kwargs):
+def wait_until(timeout, interval, delay, condition, *args, **kwargs):
     """
     @summary: Wait until the specified condition is True or timeout.
     @param timeout: Maximum time to wait
     @param interval: Poll interval
+    @param delay: Delay time
     @param condition: A function that returns False or True
     @param *args: Extra args required by the 'condition' function.
     @param **kwargs: Extra args required by the 'condition' function.
     @return: If the condition function returns True before timeout, return True. If the condition function raises an
         exception, log the error and keep waiting and polling.
     """
-    logger.debug("Wait until %s is True, timeout is %s seconds, checking interval is %s" % \
-        (condition.__name__, timeout, interval))
+    logger.debug("Wait until %s is True, timeout is %s seconds, checking interval is %s, delay is %s seconds" % \
+        (condition.__name__, timeout, interval, delay))
+
+    if delay > 0:
+        logger.debug("Delay for %s seconds first")
+        time.sleep(delay)
+
     start_time = time.time()
     elapsed_time = 0
     while elapsed_time < timeout:

tests/configlet/test_add_rack.py

@@ -138,7 +138,7 @@ def configure_dut(duthosts, rand_one_dut_hostname):
 
 def load_minigraph(duthost, duthost_name):
     config_reload(duthost, config_source="minigraph", wait=180, start_bgp=True) 
-    assert wait_until(300, 20, duthost.critical_services_fully_started), \
+    assert wait_until(300, 20, 0, duthost.critical_services_fully_started), \
             "All critical services should fully started!{}".format(duthost.critical_services)
 
 

tests/console/test_console_reversessh.py

@@ -41,7 +41,7 @@ def test_console_reversessh_connectivity(duthost, creds, target_line):
         pytest.fail("Not able to do reverse SSH to remote host via DUT")
 
     pytest_assert(
-        wait_until(10, 1, check_target_line_status, duthost, target_line, "IDLE"),
+        wait_until(10, 1, 0, check_target_line_status, duthost, target_line, "IDLE"),
         "Target line {} is busy after exited reverse SSH session".format(target_line))
 
 @pytest.mark.parametrize("target_line", ["1", "2"])
@@ -79,7 +79,7 @@ def test_console_reversessh_force_interrupt(duthost, creds, target_line):
 
     # Check the session ended within 5s and the line state is idle
     pytest_assert(
-        wait_until(5, 1, check_target_line_status, duthost, target_line, "IDLE"),
+        wait_until(5, 1, 0, check_target_line_status, duthost, target_line, "IDLE"),
         "Target line {} not toggle to IDLE state after force clear command sent")
 
     try: