diff --git a/tests/acl/test_acl.py b/tests/acl/test_acl.py index d7da769f305..5fe9c1fbaf2 100644 --- a/tests/acl/test_acl.py +++ b/tests/acl/test_acl.py @@ -534,7 +534,7 @@ def direction(self, request): def check_rule_counters(self, duthost): logger.info('Wait all rule counters are ready') - return wait_until(60, 2, self.check_rule_counters_internal, duthost) + return wait_until(60, 2, 0, self.check_rule_counters_internal, duthost) def check_rule_counters_internal(self, duthost): for asic_id in duthost.get_frontend_asic_ids(): diff --git a/tests/arp/test_neighbor_mac_noptf.py b/tests/arp/test_neighbor_mac_noptf.py index 6b4c66a1a3f..1260d735e7f 100644 --- a/tests/arp/test_neighbor_mac_noptf.py +++ b/tests/arp/test_neighbor_mac_noptf.py @@ -71,7 +71,7 @@ def setupDutConfig(self, duthosts, enum_rand_one_per_hwsku_frontend_hostname): duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname] if not duthost.get_facts().get("modular_chassis"): duthost.command("sudo config bgp shutdown all") - if not wait_until(120, 2.0, self._check_no_bgp_routes, duthost): + if not wait_until(120, 2.0, 0, self._check_no_bgp_routes, duthost): pytest.fail('BGP Shutdown Timeout: BGP route removal exceeded 120 seconds.') yield @@ -115,7 +115,7 @@ def find_routed_interface(): testRoutedInterface[asichost.asic_index] = intf return testRoutedInterface - if not wait_until(120, 2, find_routed_interface): + if not wait_until(120, 2, 0, find_routed_interface): pytest.fail('Failed to find routed interface in 120 s') yield testRoutedInterface diff --git a/tests/autorestart/test_container_autorestart.py b/tests/autorestart/test_container_autorestart.py index c02c33db94a..e8e9bcaf243 100755 --- a/tests/autorestart/test_container_autorestart.py +++ b/tests/autorestart/test_container_autorestart.py @@ -218,6 +218,7 @@ def clear_failed_flag_and_restart(duthost, container_name): duthost.shell("sudo systemctl start {}.service".format(container_name)) restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + 0, check_container_state, duthost, container_name, True) pytest_assert(restarted, "Failed to restart container '{}' after reset-failed was cleared".format(container_name)) @@ -240,6 +241,7 @@ def verify_autorestart_with_critical_process(duthost, container_name, program_na logger.info("Waiting until container '{}' is stopped...".format(container_name)) stopped = wait_until(CONTAINER_STOP_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + 0, check_container_state, duthost, container_name, False) pytest_assert(stopped, "Failed to stop container '{}'".format(container_name)) logger.info("Container '{}' was stopped".format(container_name)) @@ -247,6 +249,7 @@ def verify_autorestart_with_critical_process(duthost, container_name, program_na logger.info("Waiting until container '{}' is restarted...".format(container_name)) restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + 0, check_container_state, duthost, container_name, True) if not restarted: if is_hiting_start_limit(duthost, container_name): @@ -275,6 +278,7 @@ def verify_no_autorestart_with_non_critical_process(duthost, container_name, pro logger.info("Waiting to ensure container '{}' does not stop...".format(container_name)) stopped = wait_until(CONTAINER_STOP_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + 0, check_container_state, duthost, container_name, False) pytest_assert(not stopped, "Container '{}' was stopped unexpectedly".format(container_name)) logger.info("Container '{}' did not stop".format(container_name)) @@ -333,7 +337,7 @@ def postcheck_critical_processes_status(duthost, container_autorestart_states, u if is_hiting_start_limit(duthost, container_name): clear_failed_flag_and_restart(duthost, container_name) - return wait_until(POST_CHECK_THRESHOLD_SECS, POST_CHECK_INTERVAL_SECS, + return wait_until(POST_CHECK_THRESHOLD_SECS, POST_CHECK_INTERVAL_SECS, 0, post_test_check, duthost, up_bgp_neighbors) diff --git a/tests/bgp/bgp_helpers.py b/tests/bgp/bgp_helpers.py index 48cb87d2a48..c1670939307 100644 --- a/tests/bgp/bgp_helpers.py +++ b/tests/bgp/bgp_helpers.py @@ -44,7 +44,7 @@ def restart_bgp(duthost, asic_index=DEFAULT_ASIC_ID): duthost.asic_instance(asic_index).reset_service("bgp") duthost.asic_instance(asic_index).restart_service("bgp") docker_name = duthost.asic_instance(asic_index).get_docker_name("bgp") - pytest_assert(wait_until(100, 10, duthost.is_service_fully_started, docker_name), "BGP not started.") + pytest_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, docker_name), "BGP not started.") def define_config(duthost, template_src_path, template_dst_path): @@ -128,7 +128,7 @@ def get_routes_not_announced_to_bgpmon(duthost, ptfhost): """ def _dump_fie_exists(host): return host.stat(path=DUMP_FILE).get('stat', {}).get('exists', False) - pytest_assert(wait_until(120, 10, _dump_fie_exists, ptfhost)) + pytest_assert(wait_until(120, 10, 0, _dump_fie_exists, ptfhost)) time.sleep(20) # Wait until all routes announced to bgpmon bgpmon_routes = parse_exabgp_dump(ptfhost) rib_v4 = parse_rib(duthost, 4) diff --git a/tests/bgp/conftest.py b/tests/bgp/conftest.py index f0728869072..20ce4c347b7 100644 --- a/tests/bgp/conftest.py +++ b/tests/bgp/conftest.py @@ -137,12 +137,12 @@ def restore_nbr_gr(node=None, results=None): logger.info("bgp neighbors: {}".format(bgp_neighbors.keys())) res = True err_msg = "" - if not wait_until(300, 10, duthost.check_bgp_session_state, bgp_neighbors.keys()): + if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()): res = False err_msg = "not all bgp sessions are up after enable graceful restart" is_backend_topo = "backend" in tbinfo["topo"]["name"] - if not is_backend_topo and res and not wait_until(100, 5, duthost.check_bgp_default_route): + if not is_backend_topo and res and not wait_until(100, 5, 0, duthost.check_bgp_default_route): res = False err_msg = "ipv4 or ipv6 bgp default route not available" @@ -157,7 +157,7 @@ def restore_nbr_gr(node=None, results=None): check_results(results) - if not wait_until(300, 10, duthost.check_bgp_session_state, bgp_neighbors.keys()): + if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()): pytest.fail("not all bgp sessions are up after disable graceful restart") @@ -530,7 +530,7 @@ def bgpmon_setup_teardown(ptfhost, duthost, localhost, setup_interfaces): pt_assert(wait_tcp_connection(localhost, ptfhost.mgmt_ip, BGP_MONITOR_PORT), "Failed to start bgp monitor session on PTF") - pt_assert(wait_until(20, 5, duthost.check_bgp_session_state, [peer_addr]), 'BGP session {} on duthost is not established'.format(BGP_MONITOR_NAME)) + pt_assert(wait_until(20, 5, 0, duthost.check_bgp_session_state, [peer_addr]), 'BGP session {} on duthost is not established'.format(BGP_MONITOR_NAME)) yield # Cleanup bgp monitor diff --git a/tests/bgp/test_bgp_bbr.py b/tests/bgp/test_bgp_bbr.py index 6773a060322..d82dd436cb2 100644 --- a/tests/bgp/test_bgp_bbr.py +++ b/tests/bgp/test_bgp_bbr.py @@ -292,10 +292,10 @@ def check_other_vms(nbrhosts, setup, route, accepted=True, node=None, results=No bgp_neighbors = json.loads(duthost.shell("sonic-cfggen -d --var-json 'BGP_NEIGHBOR'")['stdout']) # check tor1 - pytest_assert(wait_until(5, 1, check_tor1, nbrhosts, setup, route), 'tor1 check failed') + pytest_assert(wait_until(5, 1, 0, check_tor1, nbrhosts, setup, route), 'tor1 check failed') # check DUT - pytest_assert(wait_until(5, 1, check_dut, duthost, other_vms, bgp_neighbors, setup, route, accepted=accepted), 'DUT check failed') + pytest_assert(wait_until(5, 1, 0, check_dut, duthost, other_vms, bgp_neighbors, setup, route, accepted=accepted), 'DUT check failed') results = parallel_run(check_other_vms, (nbrhosts, setup, route), {'accepted': accepted}, other_vms, timeout=120) diff --git a/tests/bgp/test_bgp_gr_helper.py b/tests/bgp/test_bgp_gr_helper.py index 66dc445c334..de8c1879b25 100644 --- a/tests/bgp/test_bgp_gr_helper.py +++ b/tests/bgp/test_bgp_gr_helper.py @@ -160,7 +160,7 @@ def _verify_prefix_counters_from_neighbor_after_graceful_restart(duthost, bgp_ne # wait till DUT enters NSF state for test_bgp_neighbor in test_bgp_neighbors: pytest_assert( - wait_until(60, 5, duthost.check_bgp_session_nsf, test_bgp_neighbor), + wait_until(60, 5, 0, duthost.check_bgp_session_nsf, test_bgp_neighbor), "neighbor {} does not enter NSF state".format(test_bgp_neighbor) ) @@ -187,7 +187,7 @@ def _verify_prefix_counters_from_neighbor_after_graceful_restart(duthost, bgp_ne # wait for exabgp sessions to establish pytest_assert( - wait_until(300, 10, test_neighbor_host.check_bgp_session_state, exabgp_ips, exabgp_sessions), + wait_until(300, 10, 0, test_neighbor_host.check_bgp_session_state, exabgp_ips, exabgp_sessions), "exabgp sessions {} are not coming back".format(exabgp_sessions) ) @@ -199,13 +199,13 @@ def _verify_prefix_counters_from_neighbor_after_graceful_restart(duthost, bgp_ne # confirm BGP session are up pytest_assert( - wait_until(300, 10, duthost.check_bgp_session_state, test_bgp_neighbors), + wait_until(300, 10, 0, duthost.check_bgp_session_state, test_bgp_neighbors), "graceful restarted bgp sessions {} are not coming back".format(test_bgp_neighbors) ) # confirm routes from the neighbor are restored pytest_assert( - wait_until(300, 10, _verify_prefix_counters_from_neighbor_after_graceful_restart, duthost, test_bgp_neighbors), + wait_until(300, 10, 0, _verify_prefix_counters_from_neighbor_after_graceful_restart, duthost, test_bgp_neighbors), "after graceful restart, Rib is not restored" ) diff --git a/tests/bgp/test_bgpmon.py b/tests/bgp/test_bgpmon.py index 52f80bfda08..918e7c1a6bf 100644 --- a/tests/bgp/test_bgpmon.py +++ b/tests/bgp/test_bgpmon.py @@ -151,7 +151,7 @@ def bgpmon_peer_connected(duthost, bgpmon_peer): try: pytest_assert(wait_tcp_connection(localhost, ptfhost.mgmt_ip, BGP_MONITOR_PORT), "Failed to start bgp monitor session on PTF") - pytest_assert(wait_until(180, 5, bgpmon_peer_connected, duthost, peer_addr),"BGPMon Peer connection not established") + pytest_assert(wait_until(180, 5, 0, bgpmon_peer_connected, duthost, peer_addr),"BGPMon Peer connection not established") finally: ptfhost.exabgp(name=BGP_MONITOR_NAME, state="absent") ptfhost.shell("ip route del %s dev %s" % (local_addr + "/32", ptf_interface)) diff --git a/tests/common/dualtor/control_plane_utils.py b/tests/common/dualtor/control_plane_utils.py index 3aef34a78a9..27c4d58a8fd 100644 --- a/tests/common/dualtor/control_plane_utils.py +++ b/tests/common/dualtor/control_plane_utils.py @@ -69,7 +69,7 @@ def _dump_db(self, db, key_pattern): def verify_db(self, db): pytest_assert( - wait_until(30, 10, self.get_mismatched_ports, db), + wait_until(30, 10, 0, self.get_mismatched_ports, db), "Database states don't match expected state {state}," "incorrect {db_name} values {db_states}" .format(state=self.state, db_name=DB_NAME_MAP[db], diff --git a/tests/common/dualtor/data_plane_utils.py b/tests/common/dualtor/data_plane_utils.py index fbe34527aa6..b6926fb2c1e 100644 --- a/tests/common/dualtor/data_plane_utils.py +++ b/tests/common/dualtor/data_plane_utils.py @@ -153,7 +153,7 @@ def run_test(duthosts, activehost, ptfhost, ptfadapter, action, action() # do not time-wait the test, if early stop is not requested (when stop_after=None) if stop_after is not None: - wait_until(timeout=stop_after, interval=0.5, condition=\ + wait_until(timeout=stop_after, interval=0.5, delay=0, condition=\ lambda: not send_and_sniff.is_alive) if send_and_sniff.is_alive(): logger.info("Sender/Sniffer threads are still running. Sending signal "\ diff --git a/tests/common/dualtor/mux_simulator_control.py b/tests/common/dualtor/mux_simulator_control.py index 2e21a9aac30..97cca1542b8 100644 --- a/tests/common/dualtor/mux_simulator_control.py +++ b/tests/common/dualtor/mux_simulator_control.py @@ -480,7 +480,7 @@ def _check_mux_status_consistency(): mg_facts = upper_tor_host.get_extended_minigraph_facts(tbinfo) port_indices = mg_facts['minigraph_port_indices'] pytest_assert( - utilities.wait_until(30, 5, _check_mux_status_consistency), + utilities.wait_until(30, 5, 0, _check_mux_status_consistency), "Mux status is inconsistent between the DUTs and mux simulator after toggle" ) diff --git a/tests/common/dualtor/tunnel_traffic_utils.py b/tests/common/dualtor/tunnel_traffic_utils.py index 0b4001fc112..5b47d6ccbaa 100644 --- a/tests/common/dualtor/tunnel_traffic_utils.py +++ b/tests/common/dualtor/tunnel_traffic_utils.py @@ -143,7 +143,7 @@ def _disassemble_ip_tos(tos): check_res.append("outer packet DSCP not same as inner packet DSCP") exp_queue = derive_queue_id_from_dscp(outer_dscp) - pytest_assert(wait_until(60, 5, queue_stats_check, self.standby_tor, exp_queue)) + pytest_assert(wait_until(60, 5, 0, queue_stats_check, self.standby_tor, exp_queue)) return check_res def __init__(self, standby_tor, active_tor=None, existing=True): diff --git a/tests/common/helpers/drop_counters/drop_counters.py b/tests/common/helpers/drop_counters/drop_counters.py index a41c4f06d12..61acc652886 100644 --- a/tests/common/helpers/drop_counters/drop_counters.py +++ b/tests/common/helpers/drop_counters/drop_counters.py @@ -94,7 +94,7 @@ def get_drops_across_all_duthosts(): drop_list.append(int(get_pkt_drops(duthost, get_cnt_cli_cmd, asic_index)[dut_iface][column_key].replace(",", ""))) return drop_list check_drops_on_dut = lambda: packets_count in get_drops_across_all_duthosts() - if not wait_until(25, 1, check_drops_on_dut): + if not wait_until(25, 1, 0, check_drops_on_dut): # The actual Drop count should always be equal or 1 or 2 packets more than what is expected due to some other drop may occur # over the interface being examined. When that happens if looking onlyu for exact count it will be a false positive failure. # So do one more check to allow up to 2 packets more dropped than what was expected as an allowed case. diff --git a/tests/common/helpers/dut_utils.py b/tests/common/helpers/dut_utils.py index 0591a77a23e..8f9ebd7a1d6 100644 --- a/tests/common/helpers/dut_utils.py +++ b/tests/common/helpers/dut_utils.py @@ -88,6 +88,7 @@ def clear_failed_flag_and_restart(duthost, container_name): duthost.shell("sudo systemctl start {}.service".format(container_name)) restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + 0, check_container_state, duthost, container_name, True) pytest_assert(restarted, "Failed to restart container '{}' after reset-failed was cleared".format(container_name)) diff --git a/tests/common/helpers/snmp_helpers.py b/tests/common/helpers/snmp_helpers.py index 85fb959abd7..3dd8bb46edd 100644 --- a/tests/common/helpers/snmp_helpers.py +++ b/tests/common/helpers/snmp_helpers.py @@ -35,7 +35,7 @@ def get_snmp_facts(localhost, host, version, community, is_dell=False, module_ig global global_snmp_facts - pytest_assert(wait_until(timeout, interval, _update_snmp_facts, localhost, host, version, + pytest_assert(wait_until(timeout, interval, 0, _update_snmp_facts, localhost, host, version, community, is_dell), "Timeout waiting for SNMP facts") return global_snmp_facts diff --git a/tests/common/helpers/voq_lag.py b/tests/common/helpers/voq_lag.py index 6dbd5a46d65..49df6feeb66 100644 --- a/tests/common/helpers/voq_lag.py +++ b/tests/common/helpers/voq_lag.py @@ -91,7 +91,7 @@ def add_lag(duthost, asic, portchannel_members=None, portchannel_ip=None, pytest_assert(int_facts['ansible_interface_facts'] [portchannel]['ipv4']['address'] == portchannel_ip.split('/')[0]) - pytest_assert(wait_until(30,5, verify_lag_interface, duthost, asic, portchannel), + pytest_assert(wait_until(30,5, 0, verify_lag_interface, duthost, asic, portchannel), 'For added Portchannel {} link is not up'.format(portchannel)) @@ -209,7 +209,7 @@ def delete_lag_members_ip(duthost, asic, portchannel_members, duthost.shell("config interface {} ip remove {} {}" .format(asic.cli_ns_option, portchannel, portchannel_ip)) - pytest_assert(wait_until(30,5, verify_lag_interface, duthost, asic, portchannel, expected=False), + pytest_assert(wait_until(30,5, 0, verify_lag_interface, duthost, asic, portchannel, expected=False), 'For deleted Portchannel {} ip link is not down'.format(portchannel)) diff --git a/tests/common/platform/processes_utils.py b/tests/common/platform/processes_utils.py index 3e3d71d08f1..6cd45e29d7a 100644 --- a/tests/common/platform/processes_utils.py +++ b/tests/common/platform/processes_utils.py @@ -44,6 +44,6 @@ def wait_critical_processes(dut): @param dut: The AnsibleHost object of DUT. For interacting with DUT. """ logging.info("Wait until all critical processes are healthy") - pytest_assert(wait_until(300, 20, _all_critical_processes_healthy, dut), + pytest_assert(wait_until(300, 20, 0, _all_critical_processes_healthy, dut), "Not all critical processes are healthy") diff --git a/tests/common/plugins/ptfadapter/ptfadapter.py b/tests/common/plugins/ptfadapter/ptfadapter.py index 0d6ccd30fe0..051f36b19a4 100644 --- a/tests/common/plugins/ptfadapter/ptfadapter.py +++ b/tests/common/plugins/ptfadapter/ptfadapter.py @@ -59,7 +59,7 @@ def _check_ptf_nn_agent_availability(self, socket_addr): sock = nnpy.Socket(nnpy.AF_SP, nnpy.PAIR) sock.connect(socket_addr) try: - return wait_until(1, 0.2, lambda:sock.get_statistic(self.NN_STAT_CURRENT_CONNECTIONS) == 1) + return wait_until(1, 0.2, 0, lambda:sock.get_statistic(self.NN_STAT_CURRENT_CONNECTIONS) == 1) finally: sock.close() diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py index 0a5ae1b1ed2..6e0c354c37b 100644 --- a/tests/common/plugins/sanity_check/checks.py +++ b/tests/common/plugins/sanity_check/checks.py @@ -238,7 +238,7 @@ def _check_bgp_status_helper(): networking_uptime = dut.get_networking_uptime().seconds timeout = max(SYSTEM_STABILIZE_MAX_TIME - networking_uptime, 1) interval = 20 - wait_until(timeout, interval, _check_bgp_status_helper) + wait_until(timeout, interval, 0, _check_bgp_status_helper) if (check_result['failed']): for a_result in check_result.keys(): if a_result != 'failed': diff --git a/tests/common/port_toggle.py b/tests/common/port_toggle.py index 5f5395bbf00..0ec17f6b9bc 100644 --- a/tests/common/port_toggle.py +++ b/tests/common/port_toggle.py @@ -60,7 +60,7 @@ def __get_down_ports(expect_up=True): log_system_resources(duthost, logger) logger.info("Wait for ports to go down") - shutdown_ok = wait_until(port_down_wait_time, 5, lambda: len(__get_down_ports(expect_up=False)) == len(ports)) + shutdown_ok = wait_until(port_down_wait_time, 5, 0, lambda: len(__get_down_ports(expect_up=False)) == len(ports)) if not shutdown_ok: up_ports = __get_down_ports(expect_up=True) @@ -74,7 +74,7 @@ def __get_down_ports(expect_up=True): duthost.shell_cmds(cmds=cmds_up) logger.info("Wait for ports to come up") - startup_ok = wait_until(port_up_wait_time, 5, lambda: len(__get_down_ports()) == 0) + startup_ok = wait_until(port_up_wait_time, 5, 0, lambda: len(__get_down_ports()) == 0) if not startup_ok: down_ports = __get_down_ports() diff --git a/tests/common/system_utils/docker.py b/tests/common/system_utils/docker.py index bd8c347849a..7cc7041bc68 100644 --- a/tests/common/system_utils/docker.py +++ b/tests/common/system_utils/docker.py @@ -188,7 +188,7 @@ def ready_for_swap(): return True - shutdown_check = wait_until(30, 3, ready_for_swap) + shutdown_check = wait_until(30, 3, 0, ready_for_swap) pytest_assert(shutdown_check, "Docker and/or BGP failed to shut down in 30s") @@ -196,7 +196,7 @@ def _perform_syncd_liveness_check(duthost): def check_liveness(): return duthost.is_service_running("syncd") - liveness_check = wait_until(30, 1, check_liveness) + liveness_check = wait_until(30, 1, 0, check_liveness) pytest_assert(liveness_check, "syncd crashed after swap_syncd") diff --git a/tests/common/utilities.py b/tests/common/utilities.py index e69ef6509e3..b7b3ea9df26 100644 --- a/tests/common/utilities.py +++ b/tests/common/utilities.py @@ -73,19 +73,25 @@ def wait(seconds, msg=""): time.sleep(seconds) -def wait_until(timeout, interval, condition, *args, **kwargs): +def wait_until(timeout, interval, delay, condition, *args, **kwargs): """ @summary: Wait until the specified condition is True or timeout. @param timeout: Maximum time to wait @param interval: Poll interval + @param delay: Delay time @param condition: A function that returns False or True @param *args: Extra args required by the 'condition' function. @param **kwargs: Extra args required by the 'condition' function. @return: If the condition function returns True before timeout, return True. If the condition function raises an exception, log the error and keep waiting and polling. """ - logger.debug("Wait until %s is True, timeout is %s seconds, checking interval is %s" % \ - (condition.__name__, timeout, interval)) + logger.debug("Wait until %s is True, timeout is %s seconds, checking interval is %s, delay is %s seconds" % \ + (condition.__name__, timeout, interval, delay)) + + if delay > 0: + logger.debug("Delay for %s seconds first" % delay) + time.sleep(delay) + start_time = time.time() elapsed_time = 0 while elapsed_time < timeout: diff --git a/tests/configlet/test_add_rack.py b/tests/configlet/test_add_rack.py index d2068488d79..9c2ef71ebb2 100644 --- a/tests/configlet/test_add_rack.py +++ b/tests/configlet/test_add_rack.py @@ -138,7 +138,7 @@ def configure_dut(duthosts, rand_one_dut_hostname): def load_minigraph(duthost, duthost_name): config_reload(duthost, config_source="minigraph", wait=180, start_bgp=True) - assert wait_until(300, 20, duthost.critical_services_fully_started), \ + assert wait_until(300, 20, 0, duthost.critical_services_fully_started), \ "All critical services should fully started!{}".format(duthost.critical_services) diff --git a/tests/console/test_console_reversessh.py b/tests/console/test_console_reversessh.py index 19d4a986e09..16de2353462 100644 --- a/tests/console/test_console_reversessh.py +++ b/tests/console/test_console_reversessh.py @@ -41,7 +41,7 @@ def test_console_reversessh_connectivity(duthost, creds, target_line): pytest.fail("Not able to do reverse SSH to remote host via DUT") pytest_assert( - wait_until(10, 1, check_target_line_status, duthost, target_line, "IDLE"), + wait_until(10, 1, 0, check_target_line_status, duthost, target_line, "IDLE"), "Target line {} is busy after exited reverse SSH session".format(target_line)) @pytest.mark.parametrize("target_line", ["1", "2"]) @@ -79,7 +79,7 @@ def test_console_reversessh_force_interrupt(duthost, creds, target_line): # Check the session ended within 5s and the line state is idle pytest_assert( - wait_until(5, 1, check_target_line_status, duthost, target_line, "IDLE"), + wait_until(5, 1, 0, check_target_line_status, duthost, target_line, "IDLE"), "Target line {} not toggle to IDLE state after force clear command sent") try: diff --git a/tests/container_checker/test_container_checker.py b/tests/container_checker/test_container_checker.py index 7a37cb95fc0..ca41778f1d0 100755 --- a/tests/container_checker/test_container_checker.py +++ b/tests/container_checker/test_container_checker.py @@ -152,7 +152,7 @@ def postcheck_critical_processes_status(duthost, up_bgp_neighbors): for 3 minutes. It will return False after timeout """ logger.info("Post-checking status of critical processes and BGP sessions...") - return wait_until(CONTAINER_RESTART_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + return wait_until(CONTAINER_RESTART_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, 0, post_test_check, duthost, up_bgp_neighbors) @@ -172,6 +172,7 @@ def stop_container(duthost, container_name): logger.info("Waiting until container '{}' is stopped...".format(container_name)) stopped = wait_until(CONTAINER_STOP_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + 0, check_container_state, duthost, container_name, False) pytest_assert(stopped, "Failed to stop container '{}'".format(container_name)) logger.info("Container '{}' on DuT '{}' was stopped".format(container_name, duthost.hostname)) diff --git a/tests/dhcp_relay/test_dhcp_relay.py b/tests/dhcp_relay/test_dhcp_relay.py index 788cc1e1cc7..b547b45eef2 100644 --- a/tests/dhcp_relay/test_dhcp_relay.py +++ b/tests/dhcp_relay/test_dhcp_relay.py @@ -244,7 +244,7 @@ def test_dhcp_relay_after_link_flap(ptfhost, dut_dhcp_relay_data, validate_dut_r for iface in dhcp_relay['uplink_interfaces']: duthost.shell('ifconfig {} down'.format(iface)) - pytest_assert(wait_until(50, 5, check_link_status, duthost, dhcp_relay['uplink_interfaces'], "down"), + pytest_assert(wait_until(50, 5, 0, check_link_status, duthost, dhcp_relay['uplink_interfaces'], "down"), "Not all uplinks go down") # Bring all uplink interfaces back up @@ -252,7 +252,7 @@ def test_dhcp_relay_after_link_flap(ptfhost, dut_dhcp_relay_data, validate_dut_r duthost.shell('ifconfig {} up'.format(iface)) # Wait until uplinks are up and routes are recovered - pytest_assert(wait_until(50, 5, check_routes_to_dhcp_server, duthost, dut_dhcp_relay_data), + pytest_assert(wait_until(50, 5, 0, check_routes_to_dhcp_server, duthost, dut_dhcp_relay_data), "Not all DHCP servers are routed") # Run the DHCP relay test on the PTF host @@ -295,7 +295,7 @@ def test_dhcp_relay_start_with_uplinks_down(ptfhost, dut_dhcp_relay_data, valida for iface in dhcp_relay['uplink_interfaces']: duthost.shell('ifconfig {} down'.format(iface)) - pytest_assert(wait_until(50, 5, check_link_status, duthost, dhcp_relay['uplink_interfaces'], "down"), + pytest_assert(wait_until(50, 5, 0, check_link_status, duthost, dhcp_relay['uplink_interfaces'], "down"), "Not all uplinks go down") # Restart DHCP relay service on DUT @@ -310,7 +310,7 @@ def test_dhcp_relay_start_with_uplinks_down(ptfhost, dut_dhcp_relay_data, valida duthost.shell('ifconfig {} up'.format(iface)) # Wait until uplinks are up and routes are recovered - pytest_assert(wait_until(50, 5, check_routes_to_dhcp_server, duthost, dut_dhcp_relay_data), + pytest_assert(wait_until(50, 5, 0, check_routes_to_dhcp_server, duthost, dut_dhcp_relay_data), "Not all DHCP servers are routed") # Run the DHCP relay test on the PTF host diff --git a/tests/drop_packets/test_configurable_drop_counters.py b/tests/drop_packets/test_configurable_drop_counters.py index bee50932328..3c53aae070e 100644 --- a/tests/drop_packets/test_configurable_drop_counters.py +++ b/tests/drop_packets/test_configurable_drop_counters.py @@ -331,7 +331,7 @@ def _check_drops(): recv_count, dst_port, PACKET_COUNT) return recv_count == PACKET_COUNT - pytest_assert(wait_until(10, 2, _check_drops), "Expected {} drops".format(PACKET_COUNT)) + pytest_assert(wait_until(10, 2, 0, _check_drops), "Expected {} drops".format(PACKET_COUNT)) return _runner diff --git a/tests/drop_packets/test_drop_counters.py b/tests/drop_packets/test_drop_counters.py index c68c75de858..5433ad03aab 100755 --- a/tests/drop_packets/test_drop_counters.py +++ b/tests/drop_packets/test_drop_counters.py @@ -183,7 +183,7 @@ def set_mtu(cls, mtu, iface, asic_index): raise Exception("Unsupported interface parameter - {}".format(iface)) cls.iface = iface check_mtu = lambda: get_intf_mtu(duthost, iface, asic_index) == mtu # lgtm[py/loop-variable-capture] - pytest_assert(wait_until(5, 1, check_mtu), "MTU on interface {} not updated".format(iface)) + pytest_assert(wait_until(5, 1, 0, check_mtu), "MTU on interface {} not updated".format(iface)) cls.asic_index = asic_index @classmethod diff --git a/tests/fdb/test_fdb.py b/tests/fdb/test_fdb.py index 87c13ae047b..1e72364fa33 100644 --- a/tests/fdb/test_fdb.py +++ b/tests/fdb/test_fdb.py @@ -226,7 +226,7 @@ def fdb_cleanup(duthosts, rand_one_dut_hostname): return else: duthost.command('sonic-clear fdb all') - pytest_assert(wait_until(20, 2, fdb_table_has_no_dynamic_macs, duthost), "FDB Table Cleanup failed") + pytest_assert(wait_until(20, 2, 0, fdb_table_has_no_dynamic_macs, duthost), "FDB Table Cleanup failed") def validate_mac(mac): diff --git a/tests/iface_namingmode/test_iface_namingmode.py b/tests/iface_namingmode/test_iface_namingmode.py index a346cd9f084..f9787dc4c37 100644 --- a/tests/iface_namingmode/test_iface_namingmode.py +++ b/tests/iface_namingmode/test_iface_namingmode.py @@ -688,14 +688,14 @@ def _port_status(expected_state): ifmode, cli_ns_option, test_intf)) if out['rc'] != 0: pytest.fail() - pytest_assert(wait_until(PORT_TOGGLE_TIMEOUT, 2, _port_status, 'down'), + pytest_assert(wait_until(PORT_TOGGLE_TIMEOUT, 2, 0, _port_status, 'down'), "Interface {} should be admin down".format(test_intf)) out = dutHostGuest.shell('SONIC_CLI_IFACE_MODE={} sudo config interface {} startup {}'.format( ifmode, cli_ns_option, test_intf)) if out['rc'] != 0: pytest.fail() - pytest_assert(wait_until(PORT_TOGGLE_TIMEOUT, 2, _port_status, 'up'), + pytest_assert(wait_until(PORT_TOGGLE_TIMEOUT, 2, 0, _port_status, 'up'), "Interface {} should be admin up".format(test_intf)) diff --git a/tests/ixia/pfc/test_pfc_pause_lossless.py b/tests/ixia/pfc/test_pfc_pause_lossless.py index 46d0b46493e..21fbf98e8c4 100644 --- a/tests/ixia/pfc/test_pfc_pause_lossless.py +++ b/tests/ixia/pfc/test_pfc_pause_lossless.py @@ -181,7 +181,7 @@ def test_pfc_pause_single_lossless_prio_reboot(ixia_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=ixia_api, @@ -248,7 +248,7 @@ def test_pfc_pause_multi_lossless_prio_reboot(ixia_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=ixia_api, diff --git a/tests/ixia/pfc/test_pfc_pause_lossy.py b/tests/ixia/pfc/test_pfc_pause_lossy.py index 136f300c940..cd3b5a19c54 100644 --- a/tests/ixia/pfc/test_pfc_pause_lossy.py +++ b/tests/ixia/pfc/test_pfc_pause_lossy.py @@ -181,7 +181,7 @@ def test_pfc_pause_single_lossy_prio_reboot(ixia_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=ixia_api, @@ -248,7 +248,7 @@ def test_pfc_pause_multi_lossy_prio_reboot(ixia_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=ixia_api, diff --git a/tests/ixia/pfcwd/test_pfcwd_basic.py b/tests/ixia/pfcwd/test_pfcwd_basic.py index a3039f8a7be..bf5c6036257 100644 --- a/tests/ixia/pfcwd/test_pfcwd_basic.py +++ b/tests/ixia/pfcwd/test_pfcwd_basic.py @@ -169,7 +169,7 @@ def test_pfcwd_basic_single_lossless_prio_reboot(ixia_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfcwd_basic_test(api=ixia_api, @@ -232,7 +232,7 @@ def test_pfcwd_basic_multi_lossless_prio_reboot(ixia_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfcwd_basic_test(api=ixia_api, @@ -296,7 +296,7 @@ def test_pfcwd_basic_single_lossless_prio_service_restart(ixia_api, duthost.command("systemctl reset-failed {}".format(service)) duthost.command("systemctl restart {}".format(restart_service)) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfcwd_basic_test(api=ixia_api, @@ -359,7 +359,7 @@ def test_pfcwd_basic_multi_lossless_prio_restart_service(ixia_api, duthost.command("systemctl reset-failed {}".format(service)) duthost.command("systemctl restart {}".format(restart_service)) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfcwd_basic_test(api=ixia_api, diff --git a/tests/memory_checker/test_memory_checker.py b/tests/memory_checker/test_memory_checker.py index 82194403546..573437910c5 100644 --- a/tests/memory_checker/test_memory_checker.py +++ b/tests/memory_checker/test_memory_checker.py @@ -159,6 +159,7 @@ def consume_memory_and_restart_container(duthost, container_name, vm_workers, lo logger.info("Waiting for '{}' container to be restarted ...".format(container_name)) restarted = wait_until(CONTAINER_RESTART_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + 0, check_container_state, duthost, container_name, True) pytest_assert(restarted, "Failed to restart '{}' container!".format(container_name)) logger.info("'{}' container is restarted.".format(container_name)) @@ -193,7 +194,7 @@ def postcheck_critical_processes(duthost, container_name): """ logger.info("Checking the running status of critical processes in '{}' container ..." .format(container_name)) - is_succeeded = wait_until(CONTAINER_RESTART_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, + is_succeeded = wait_until(CONTAINER_RESTART_THRESHOLD_SECS, CONTAINER_CHECK_INTERVAL_SECS, 0, check_critical_processes, duthost, container_name) if not is_succeeded: pytest.fail("Not all critical processes in '{}' container are running!" diff --git a/tests/monit/test_monit_status.py b/tests/monit/test_monit_status.py index 60c3dcb407d..50a3e16ff39 100644 --- a/tests/monit/test_monit_status.py +++ b/tests/monit/test_monit_status.py @@ -105,7 +105,7 @@ def _monit_status(): monit_status_result = duthost.shell("sudo monit status", module_ignore_errors=True) return monit_status_result["rc"] == 0 # Monit is configured with start delay = 300s, hence we wait up to 320s here - pytest_assert(wait_until(320, 20, _monit_status), + pytest_assert(wait_until(320, 20, 0, _monit_status), "Monit is either not running or not configured correctly") logger.info("Checking the running status of Monit was done!") @@ -133,7 +133,7 @@ def test_monit_reporting_message(duthosts, enum_rand_one_per_hwsku_frontend_host logger.info("Checking the format of Monit alerting message ...") - pytest_assert(wait_until(180, 60, check_monit_last_output, duthost), + pytest_assert(wait_until(180, 60, 0, check_monit_last_output, duthost), "Expected Monit reporting message not found") logger.info("Checking the format of Monit alerting message was done!") diff --git a/tests/mvrf/test_mgmtvrf.py b/tests/mvrf/test_mgmtvrf.py index 4f93281b133..b111649e7c6 100644 --- a/tests/mvrf/test_mgmtvrf.py +++ b/tests/mvrf/test_mgmtvrf.py @@ -138,7 +138,7 @@ def setup_ntp(ptfhost, duthost, ntp_servers): ptfhost.lineinfile(path="/etc/ntp.conf", line="server 127.127.1.0 prefer") # restart ntp server ntp_en_res = ptfhost.service(name="ntp", state="restarted") - pytest_assert(wait_until(120, 5, check_ntp_status, ptfhost), \ + pytest_assert(wait_until(120, 5, 0, check_ntp_status, ptfhost), \ "NTP server was not started in PTF container {}; NTP service start result {}".format(ptfhost.hostname, ntp_en_res)) # setup ntp on dut to sync with ntp server for ntp_server in ntp_servers: @@ -207,7 +207,7 @@ def test_ntp(self, duthosts, rand_one_dut_hostname, ptfhost, check_ntp_sync, ntp logger.info("Ntp restart in mgmt vrf") execute_dut_command(duthost, force_ntp) duthost.service(name="ntp", state="restarted") - pytest_assert(wait_until(400, 10, check_ntp_status, duthost), "Ntp not started") + pytest_assert(wait_until(400, 10, 0, check_ntp_status, duthost), "Ntp not started") def test_service_acl(self, duthosts, rand_one_dut_hostname, localhost): duthost = duthosts[rand_one_dut_hostname] @@ -247,7 +247,7 @@ def test_warmboot(self, duthosts, rand_one_dut_hostname, localhost, ptfhost, cre duthost = duthosts[rand_one_dut_hostname] duthost.command("sudo config save -y") # This will override config_db.json with mgmt vrf config reboot(duthost, localhost, reboot_type="warm") - pytest_assert(wait_until(120, 20, duthost.critical_services_fully_started), "Not all critical services are fully started") + pytest_assert(wait_until(120, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") self.basic_check_after_reboot(duthost, localhost, ptfhost, creds) @pytest.mark.disable_loganalyzer @@ -255,7 +255,7 @@ def test_reboot(self, duthosts, rand_one_dut_hostname, localhost, ptfhost, creds duthost = duthosts[rand_one_dut_hostname] duthost.command("sudo config save -y") # This will override config_db.json with mgmt vrf config reboot(duthost, localhost) - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started") + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") self.basic_check_after_reboot(duthost, localhost, ptfhost, creds) @pytest.mark.disable_loganalyzer @@ -263,5 +263,5 @@ def test_fastboot(self, duthosts, rand_one_dut_hostname, localhost, ptfhost, cre duthost = duthosts[rand_one_dut_hostname] duthost.command("sudo config save -y") # This will override config_db.json with mgmt vrf config reboot(duthost, localhost, reboot_type="fast") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started") + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") self.basic_check_after_reboot(duthost, localhost, ptfhost, creds) diff --git a/tests/ntp/test_ntp.py b/tests/ntp/test_ntp.py index 60f4b7ea459..9e3cc392205 100644 --- a/tests/ntp/test_ntp.py +++ b/tests/ntp/test_ntp.py @@ -22,7 +22,7 @@ def setup_ntp(ptfhost, duthosts, rand_one_dut_hostname, creds): # restart ntp server ntp_en_res = ptfhost.service(name="ntp", state="restarted") - pytest_assert(wait_until(120, 5, check_ntp_status, ptfhost), \ + pytest_assert(wait_until(120, 5, 0, check_ntp_status, ptfhost), \ "NTP server was not started in PTF container {}; NTP service start result {}".format(ptfhost.hostname, ntp_en_res)) # setup ntp on dut to sync with ntp server @@ -57,5 +57,5 @@ def test_ntp(duthosts, rand_one_dut_hostname, setup_ntp): duthost.service(name='ntp', state='stopped') duthost.command("ntpd -gq") duthost.service(name='ntp', state='restarted') - pytest_assert(wait_until(720, 10, check_ntp_status, duthost), + pytest_assert(wait_until(720, 10, 0, check_ntp_status, duthost), "NTP not in sync") diff --git a/tests/pc/test_lag_2.py b/tests/pc/test_lag_2.py index 0940d7a5063..dbba0b5b85f 100644 --- a/tests/pc/test_lag_2.py +++ b/tests/pc/test_lag_2.py @@ -112,7 +112,7 @@ def __verify_lag_minlink( if po_intf != intf: command = 'bash -c "teamdctl %s %s state dump" | python -c "import sys, json; print json.load(sys.stdin)[\'ports\'][\'%s\'][\'runner\'][\'selected\']"' \ % (namespace_prefix, lag_name, po_intf) - wait_until(wait_timeout, delay, self.__check_shell_output, self.duthost, command) + wait_until(wait_timeout, delay, 0, self.__check_shell_output, self.duthost, command) # Refresh lag facts lag_facts = self.__get_lag_facts() @@ -136,7 +136,7 @@ def __verify_lag_minlink( if po_intf != intf: command = 'bash -c "teamdctl %s %s state dump" | python -c "import sys, json; print json.load(sys.stdin)[\'ports\'][\'%s\'][\'link\'][\'up\']"'\ % (namespace_prefix, lag_name, po_intf) - wait_until(wait_timeout, delay, self.__check_shell_output, self.duthost, command) + wait_until(wait_timeout, delay, 0, self.__check_shell_output, self.duthost, command) def run_single_lag_lacp_rate_test(self, lag_name, lag_facts): logger.info("Start checking single lag lacp rate for: %s" % lag_name) @@ -216,7 +216,7 @@ def run_lag_fallback_test(self, lag_name, lag_facts): try: # Shut down neighbor interface vm_host.shutdown(neighbor_intf) - wait_until(wait_timeout, delay, self.__check_intf_state, vm_host, neighbor_intf, False) + wait_until(wait_timeout, delay, 0, self.__check_intf_state, vm_host, neighbor_intf, False) # Refresh lag facts lag_facts = self.__get_lag_facts() @@ -242,7 +242,7 @@ def run_lag_fallback_test(self, lag_name, lag_facts): finally: # Bring up neighbor interface vm_host.no_shutdown(neighbor_intf) - wait_until(wait_timeout, delay, self.__check_intf_state, vm_host, neighbor_intf, True) + wait_until(wait_timeout, delay, 0, self.__check_intf_state, vm_host, neighbor_intf, True) @pytest.fixture(autouse=True, scope='module') def skip_if_no_lags(duthosts): diff --git a/tests/pc/test_po_cleanup.py b/tests/pc/test_po_cleanup.py index ed710741edf..6da34ad6299 100644 --- a/tests/pc/test_po_cleanup.py +++ b/tests/pc/test_po_cleanup.py @@ -59,6 +59,6 @@ def test_po_cleanup(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_as logging.info("Disable swss/teamd Feature") duthost.asic_instance(enum_asic_index).stop_service("swss") # Check if Linux Kernel Portchannel Interface teamdev are clean up - if not wait_until(10, 1, check_kernel_po_interface_cleaned, duthost, enum_asic_index): + if not wait_until(10, 1, 0, check_kernel_po_interface_cleaned, duthost, enum_asic_index): fail_msg = "PortChannel interface still exists in kernel" pytest.fail(fail_msg) diff --git a/tests/pc/test_po_update.py b/tests/pc/test_po_update.py index 2c9e3982595..dbad98e1b27 100644 --- a/tests/pc/test_po_update.py +++ b/tests/pc/test_po_update.py @@ -76,7 +76,7 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro time.sleep(30) int_facts = asichost.interface_facts()['ansible_facts'] pytest_assert(not int_facts['ansible_interface_facts'][portchannel]['link']) - pytest_assert(wait_until(120, 10, asichost.check_bgp_statistic, 'ipv4_idle', 1)) + pytest_assert(wait_until(120, 10, 0, asichost.check_bgp_statistic, 'ipv4_idle', 1)) # Step 3: Create tmp portchannel asichost.config_portchannel(tmp_portchannel, "add") @@ -96,7 +96,7 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro time.sleep(30) int_facts = asichost.interface_facts()['ansible_facts'] pytest_assert(int_facts['ansible_interface_facts'][tmp_portchannel]['link']) - pytest_assert(wait_until(120, 10, asichost.check_bgp_statistic, 'ipv4_idle', 0)) + pytest_assert(wait_until(120, 10, 0, asichost.check_bgp_statistic, 'ipv4_idle', 0)) finally: # Recover all states if add_tmp_portchannel_ip: @@ -115,4 +115,4 @@ def test_po_update(duthosts, enum_rand_one_per_hwsku_frontend_hostname, enum_fro if remove_portchannel_members: for member in portchannel_members: asichost.config_portchannel_member(portchannel, member, "add") - pytest_assert(wait_until(120, 10, asichost.check_bgp_statistic, 'ipv4_idle', 0)) + pytest_assert(wait_until(120, 10, 0, asichost.check_bgp_statistic, 'ipv4_idle', 0)) diff --git a/tests/platform_tests/api/test_sfp.py b/tests/platform_tests/api/test_sfp.py index 10715a9c2f1..4d8645bfdbe 100644 --- a/tests/platform_tests/api/test_sfp.py +++ b/tests/platform_tests/api/test_sfp.py @@ -509,7 +509,7 @@ def test_lpmode(self, duthosts, enum_rand_one_per_hwsku_hostname, localhost, pla logger.warning("test_lpmode: Skipping transceiver {} (not supported on this platform)".format(i)) break self.expect(ret is True, "Failed to {} low-power mode for transceiver {}".format("enable" if state is True else "disable", i)) - self.expect(wait_until(5, 1, self._check_lpmode_status, sfp, platform_api_conn, i, state), + self.expect(wait_until(5, 1, 0, self._check_lpmode_status, sfp, platform_api_conn, i, state), "Transceiver {} expected low-power state {} is not aligned with the real state".format(i, "enable" if state is True else "disable")) self.assert_expectations() diff --git a/tests/platform_tests/daemon/test_ledd.py b/tests/platform_tests/daemon/test_ledd.py index 2aa56f2af8a..992b43f062b 100644 --- a/tests/platform_tests/daemon/test_ledd.py +++ b/tests/platform_tests/daemon/test_ledd.py @@ -132,7 +132,7 @@ def test_pmon_ledd_term_and_start_status(check_daemon_status, duthosts, rand_one # TODO: To arm the wait_until API with a delay parameter, by which to delay specified time # before invoking the check function. - wait_until(120, 10, check_expected_daemon_status, duthost, expected_running_status) + wait_until(120, 10, 0, check_expected_daemon_status, duthost, expected_running_status) post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name) pytest_assert(post_daemon_status == expected_running_status, diff --git a/tests/platform_tests/daemon/test_pcied.py b/tests/platform_tests/daemon/test_pcied.py index ea3e6856514..a5e93cc958e 100644 --- a/tests/platform_tests/daemon/test_pcied.py +++ b/tests/platform_tests/daemon/test_pcied.py @@ -98,7 +98,7 @@ def _collect_data(): shared_scope.data_after_restart = collect_data(duthost) return bool(shared_scope.data_after_restart['devices']) pcied_pooling_interval = 60 - wait_until(pcied_pooling_interval, 6, _collect_data) + wait_until(pcied_pooling_interval, 6, 0, _collect_data) return shared_scope.data_after_restart @pytest.fixture(scope='module') diff --git a/tests/platform_tests/daemon/test_psud.py b/tests/platform_tests/daemon/test_psud.py index da684d8ef99..61a56e6bf64 100644 --- a/tests/platform_tests/daemon/test_psud.py +++ b/tests/platform_tests/daemon/test_psud.py @@ -92,7 +92,7 @@ def _collect_data(): shared_scope.data_after_restart = collect_data(duthost) return bool(shared_scope.data_after_restart['data']) psud_pooling_interval = 60 - wait_until(psud_pooling_interval, 6, _collect_data) + wait_until(psud_pooling_interval, 6, 0, _collect_data) return shared_scope.data_after_restart @pytest.fixture(scope='module') @@ -142,7 +142,7 @@ def test_pmon_psud_stop_and_start_status(check_daemon_status, duthosts, rand_one duthost.start_pmon_daemon(daemon_name) - wait_until(50, 10, check_expected_daemon_status, duthost, expected_running_status) + wait_until(50, 10, 0, check_expected_daemon_status, duthost, expected_running_status) post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name) pytest_assert(post_daemon_status == expected_running_status, @@ -169,7 +169,7 @@ def test_pmon_psud_term_and_start_status(check_daemon_status, duthosts, rand_one duthost.stop_pmon_daemon(daemon_name, SIG_TERM, pre_daemon_pid) - wait_until(50, 10, check_expected_daemon_status, duthost, expected_running_status) + wait_until(50, 10, 0, check_expected_daemon_status, duthost, expected_running_status) post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name) pytest_assert(post_daemon_status == expected_running_status, @@ -195,7 +195,7 @@ def test_pmon_psud_kill_and_start_status(check_daemon_status, duthosts, rand_one duthost.stop_pmon_daemon(daemon_name, SIG_KILL, pre_daemon_pid) - wait_until(120, 10, check_expected_daemon_status, duthost, expected_running_status) + wait_until(120, 10, 0, check_expected_daemon_status, duthost, expected_running_status) post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name) pytest_assert(post_daemon_status == expected_running_status, diff --git a/tests/platform_tests/daemon/test_syseepromd.py b/tests/platform_tests/daemon/test_syseepromd.py index 21829a6005a..2d74ad1f5a0 100644 --- a/tests/platform_tests/daemon/test_syseepromd.py +++ b/tests/platform_tests/daemon/test_syseepromd.py @@ -92,7 +92,7 @@ def _collect_data(): shared_scope.data_after_restart = collect_data(duthost) return bool(shared_scope.data_after_restart['data']) syseepromd_pooling_interval = 60 - wait_until(syseepromd_pooling_interval, 6, _collect_data) + wait_until(syseepromd_pooling_interval, 6, 0, _collect_data) return shared_scope.data_after_restart @pytest.fixture(scope='module') @@ -168,7 +168,7 @@ def test_pmon_syseepromd_term_and_start_status(check_daemon_status, duthosts, ra duthost.stop_pmon_daemon(daemon_name, SIG_TERM, pre_daemon_pid) - wait_until(50, 10, check_expected_daemon_status, duthost, expected_running_status) + wait_until(50, 10, 0, check_expected_daemon_status, duthost, expected_running_status) post_daemon_status, post_daemon_pid = duthost.get_pmon_daemon_status(daemon_name) pytest_assert(post_daemon_status == expected_running_status, diff --git a/tests/platform_tests/fwutil/conftest.py b/tests/platform_tests/fwutil/conftest.py index f9e51eba3a2..8e1f82ecc29 100644 --- a/tests/platform_tests/fwutil/conftest.py +++ b/tests/platform_tests/fwutil/conftest.py @@ -91,7 +91,7 @@ def next_image(duthost, fw_pkg): logger.info("Attempting to stage test firware onto newly-installed image.") try: - wait_until(10, 1, check_path_exists, fs_rw) + wait_until(10, 1, 0, check_path_exists, fs_rw) duthost.command("mkdir -p {}".format(fs_mountpoint)) cmd = "mount -t squashfs {} {}".format(fs_path, fs_mountpoint) diff --git a/tests/platform_tests/fwutil/fwutil_common.py b/tests/platform_tests/fwutil/fwutil_common.py index 0b5c9ac7796..cb6c668c5b0 100644 --- a/tests/platform_tests/fwutil/fwutil_common.py +++ b/tests/platform_tests/fwutil/fwutil_common.py @@ -71,7 +71,7 @@ def complete_install(duthost, localhost, boot_type, res, pdu_ctrl, auto_reboot=F logger.info("Waiting on switch to come up....") localhost.wait_for(host=hn, port=22, state='started', delay=10, timeout=300) logger.info("Waiting on critical systems to come online...") - wait_until(300, 30, duthost.critical_services_fully_started) + wait_until(300, 30, 0, duthost.critical_services_fully_started) time.sleep(60) # Reboot back into original image if neccesary @@ -84,7 +84,7 @@ def complete_install(duthost, localhost, boot_type, res, pdu_ctrl, auto_reboot=F time.sleep(100) logger.info("Waiting on switch to come up....") localhost.wait_for(host=hn, port=22, state='started', delay=10, timeout=150) - wait_until(300, 30, duthost.critical_services_fully_started) + wait_until(300, 30, 0, duthost.critical_services_fully_started) time.sleep(60) def show_firmware(duthost): diff --git a/tests/platform_tests/link_flap/link_flap_utils.py b/tests/platform_tests/link_flap/link_flap_utils.py index e1eeb7463e0..524fd445ee3 100644 --- a/tests/platform_tests/link_flap/link_flap_utils.py +++ b/tests/platform_tests/link_flap/link_flap_utils.py @@ -133,7 +133,7 @@ def toggle_one_link(dut, dut_port, fanout, fanout_port, watch=False): need_recovery = True try: fanout.shutdown(fanout_port) - pytest_assert(wait_until(30, 1, __check_if_status, dut, dut_port, 'down', True), "dut port {} didn't go down as expected".format(dut_port)) + pytest_assert(wait_until(30, 1, 0, __check_if_status, dut, dut_port, 'down', True), "dut port {} didn't go down as expected".format(dut_port)) if watch: time.sleep(1) @@ -142,11 +142,11 @@ def toggle_one_link(dut, dut_port, fanout, fanout_port, watch=False): logger.info("Bring up fanout switch %s port %s connecting to %s", fanout.hostname, fanout_port, dut_port) fanout.no_shutdown(fanout_port) need_recovery = False - pytest_assert(wait_until(30, 1, __check_if_status, dut, dut_port, 'up', True), "dut port {} didn't go up as expected".format(dut_port)) + pytest_assert(wait_until(30, 1, 0, __check_if_status, dut, dut_port, 'up', True), "dut port {} didn't go up as expected".format(dut_port)) finally: if need_recovery: fanout.no_shutdown(fanout_port) - wait_until(30, 1, __check_if_status, dut, dut_port, 'up', True) + wait_until(30, 1, 0, __check_if_status, dut, dut_port, 'up', True) def watch_system_status(dut): diff --git a/tests/platform_tests/link_flap/test_cont_link_flap.py b/tests/platform_tests/link_flap/test_cont_link_flap.py index 4eb731702b4..d939a5fbe93 100644 --- a/tests/platform_tests/link_flap/test_cont_link_flap.py +++ b/tests/platform_tests/link_flap/test_cont_link_flap.py @@ -63,7 +63,7 @@ def test_cont_link_flap(self, request, duthosts, enum_rand_one_per_hwsku_fronten # Make Sure Orch CPU < orch_cpu_threshold before starting test. logging.info("Make Sure orchagent CPU utilization is less that %d before link flap", orch_cpu_threshold) - pytest_assert(wait_until(100, 2, check_orch_cpu_utilization, duthost, orch_cpu_threshold), + pytest_assert(wait_until(100, 2, 0, check_orch_cpu_utilization, duthost, orch_cpu_threshold), "Orch CPU utilization {} > orch cpu threshold {} before link flap" .format(duthost.shell("show processes cpu | grep orchagent | awk '{print $9}'")["stdout"], orch_cpu_threshold)) @@ -83,7 +83,7 @@ def test_cont_link_flap(self, request, duthosts, enum_rand_one_per_hwsku_fronten toggle_one_link(duthost, dut_port, fanout, fanout_port, watch=True) # Make Sure all ipv4/ipv6 routes are relearned with jitter of ~5 - if not wait_until(60, 1, check_bgp_routes, duthost, start_time_ipv4_route_counts, start_time_ipv6_route_counts): + if not wait_until(60, 1, 0, check_bgp_routes, duthost, start_time_ipv4_route_counts, start_time_ipv6_route_counts): endv4, endv6 = duthost.get_ip_route_summary() pytest.fail("IP routes are not equal after link flap: before ipv4 {} ipv6 {}, after ipv4 {} ipv6 {}".format(sumv4, sumv6, endv4, endv6)) @@ -112,6 +112,6 @@ def test_cont_link_flap(self, request, duthosts, enum_rand_one_per_hwsku_fronten # Orchagent CPU should consume < orch_cpu_threshold at last. logging.info("watch orchagent CPU utilization when it goes below %d", orch_cpu_threshold) - pytest_assert(wait_until(45, 2, check_orch_cpu_utilization, duthost, orch_cpu_threshold), + pytest_assert(wait_until(45, 2, 0, check_orch_cpu_utilization, duthost, orch_cpu_threshold), "Orch CPU utilization {} > orch cpu threshold {} before link flap" .format(duthost.shell("show processes cpu | grep orchagent | awk '{print $9}'")["stdout"], orch_cpu_threshold)) diff --git a/tests/platform_tests/mellanox/check_sysfs.py b/tests/platform_tests/mellanox/check_sysfs.py index c7ced758804..b6b70bdeb1e 100644 --- a/tests/platform_tests/mellanox/check_sysfs.py +++ b/tests/platform_tests/mellanox/check_sysfs.py @@ -67,7 +67,7 @@ def check_sysfs(dut): if not _is_fan_speed_in_range(sysfs_facts): sysfs_fan_config = [generate_sysfs_fan_config(platform_data)] - assert wait_until(30, 5, _check_fan_speed_in_range, dut, sysfs_fan_config), "Fan speed not in range" + assert wait_until(30, 5, 0, _check_fan_speed_in_range, dut, sysfs_fan_config), "Fan speed not in range" logging.info("Check CPU related sysfs") cpu_temp_high_counter = 0 diff --git a/tests/platform_tests/mellanox/test_thermal_control.py b/tests/platform_tests/mellanox/test_thermal_control.py index ad7140c8932..d39a05316a6 100644 --- a/tests/platform_tests/mellanox/test_thermal_control.py +++ b/tests/platform_tests/mellanox/test_thermal_control.py @@ -47,6 +47,7 @@ def test_dynamic_minimum_table(duthosts, rand_one_dut_hostname, mocker_factory): mocker.mock_min_table(temperature, trust_state) assert wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, + 0, check_cooling_level_larger_than_minimum, duthost, expect_minimum_cooling_level), \ @@ -59,6 +60,7 @@ def test_dynamic_minimum_table(duthosts, rand_one_dut_hostname, mocker_factory): mocker.mock_min_table(temperature, not trust_state) assert wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, + 0, check_cooling_level_larger_than_minimum, duthost, expect_minimum_cooling_level), \ @@ -82,6 +84,7 @@ def test_set_psu_fan_speed(duthosts, rand_one_dut_hostname, mocker_factory): single_fan_mocker.mock_absence() assert wait_until(THERMAL_CONTROL_TEST_WAIT_TIME * 2, THERMAL_CONTROL_TEST_CHECK_INTERVAL, + 0, check_psu_fan_speed, duthost, psu_num, @@ -92,6 +95,7 @@ def test_set_psu_fan_speed(duthosts, rand_one_dut_hostname, mocker_factory): single_fan_mocker.mock_presence() wait_result = wait_until(THERMAL_CONTROL_TEST_WAIT_TIME * 2, THERMAL_CONTROL_TEST_CHECK_INTERVAL, + 0, check_psu_fan_speed, duthost, psu_num, diff --git a/tests/platform_tests/test_auto_negotiation.py b/tests/platform_tests/test_auto_negotiation.py index edb7b548820..c0b995e8905 100644 --- a/tests/platform_tests/test_auto_negotiation.py +++ b/tests/platform_tests/test_auto_negotiation.py @@ -231,6 +231,7 @@ def test_auto_negotiation_advertised_speeds_all(): logger.info('Wait until all ports are up') wait_result = wait_until(ALL_PORT_WAIT_TIME, PORT_STATUS_CHECK_INTERVAL, + 0, check_ports_up, duthost, [item[1] for item in candidates.values()]) @@ -290,6 +291,7 @@ def test_auto_negotiation_dut_advertises_each_speed(enum_dut_portname_module_fix logger.info('Wait until the port status is up, expected speed: {}'.format(speed)) wait_result = wait_until(SINGLE_PORT_WAIT_TIME, PORT_STATUS_CHECK_INTERVAL, + 0, check_ports_up, duthost, [dut_port], @@ -337,6 +339,7 @@ def test_auto_negotiation_fanout_advertises_each_speed(enum_dut_portname_module_ logger.info('Wait until the port status is up, expected speed: {}'.format(speed)) wait_result = wait_until(SINGLE_PORT_WAIT_TIME, PORT_STATUS_CHECK_INTERVAL, + 0, check_ports_up, duthost, [dut_port], @@ -382,6 +385,7 @@ def test_force_speed(enum_dut_portname_module_fixture): logger.info('Wait until the port status is up, expected speed: {}'.format(speed)) wait_result = wait_until(SINGLE_PORT_WAIT_TIME, PORT_STATUS_CHECK_INTERVAL, + 0, check_ports_up, duthost, [dut_port], diff --git a/tests/platform_tests/test_platform_info.py b/tests/platform_tests/test_platform_info.py index 98ee6dbd6fa..c1292ef7127 100644 --- a/tests/platform_tests/test_platform_info.py +++ b/tests/platform_tests/test_platform_info.py @@ -366,6 +366,7 @@ def test_thermal_control_psu_absence(duthosts, enum_rand_one_per_hwsku_hostname, logging.info('Wait and check all FAN speed turn to 60%...') wait_result = wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, + 0, fan_mocker.check_all_fan_speed, 60) @@ -390,6 +391,7 @@ def test_thermal_control_psu_absence(duthosts, enum_rand_one_per_hwsku_hostname, logging.info('Wait and check all FAN speed turn to 65%...') pytest_assert(wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, + 0, fan_mocker.check_all_fan_speed, 65), 'FAN speed not change to 65% according to policy') @@ -416,6 +418,7 @@ def turn_off_outlet_and_check_thermal_control(dut, pdu_ctrl, outlet, mocker): logging.info('Wait and check all FAN speed turn to 100%...') pytest_assert(wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, + 0, mocker.check_all_fan_speed, 100), 'FAN speed not turn to 100% after PSU off') @@ -439,7 +442,7 @@ def test_thermal_control_fan_status(duthosts, enum_rand_one_per_hwsku_hostname, logging.info('Mock FAN status data...') fan_mocker.mock_data() # make data random restart_thermal_control_daemon(duthost) - wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, fan_mocker.check_all_fan_speed, + wait_until(THERMAL_CONTROL_TEST_WAIT_TIME, THERMAL_CONTROL_TEST_CHECK_INTERVAL, 0, fan_mocker.check_all_fan_speed, 60) check_thermal_algorithm_status(duthost, mocker_factory, False) diff --git a/tests/platform_tests/test_reboot.py b/tests/platform_tests/test_reboot.py index ae2e92471a1..51ca3a52646 100644 --- a/tests/platform_tests/test_reboot.py +++ b/tests/platform_tests/test_reboot.py @@ -75,14 +75,14 @@ def check_interfaces_and_services(dut, interfaces, xcvr_skip_list, reboot_type = if reboot_type is not None: logging.info("Check reboot cause") - assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause, dut, reboot_type), \ + assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, 0, check_reboot_cause, dut, reboot_type), \ "got reboot-cause failed after rebooted by %s" % reboot_type if "201811" in dut.os_version or "201911" in dut.os_version: logging.info("Skip check reboot-cause history for version before 202012") else: logger.info("Check reboot-cause history") - assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, check_reboot_cause_history, dut, + assert wait_until(MAX_WAIT_TIME_FOR_REBOOT_CAUSE, 20, 0, check_reboot_cause_history, dut, REBOOT_TYPE_HISTOYR_QUEUE), "Check reboot-cause history failed after rebooted by %s" % reboot_type if reboot_ctrl_dict[reboot_type]["test_reboot_cause_only"]: logging.info("Further checking skipped for %s test which intends to verify reboot-cause only" % reboot_type) @@ -92,7 +92,7 @@ def check_interfaces_and_services(dut, interfaces, xcvr_skip_list, reboot_type = logging.info("skipping interfaces related check for supervisor") else: logging.info("Wait {} seconds for all the transceivers to be detected".format(MAX_WAIT_TIME_FOR_INTERFACES)) - result = wait_until(MAX_WAIT_TIME_FOR_INTERFACES, 20, check_all_interface_information, dut, interfaces, + result = wait_until(MAX_WAIT_TIME_FOR_INTERFACES, 20, 0, check_all_interface_information, dut, interfaces, xcvr_skip_list) assert result, "Not all transceivers are detected or interfaces are up in {} seconds".format( MAX_WAIT_TIME_FOR_INTERFACES) diff --git a/tests/platform_tests/test_reload_config.py b/tests/platform_tests/test_reload_config.py index 4d8cc9c992a..2c76bc50a46 100644 --- a/tests/platform_tests/test_reload_config.py +++ b/tests/platform_tests/test_reload_config.py @@ -30,7 +30,7 @@ def test_reload_configuration(duthosts, rand_one_dut_hostname, conn_graph_facts, asic_type = duthost.facts["asic_type"] if config_force_option_supported(duthost): - assert wait_until(300, 20, config_system_checks_passed, duthost) + assert wait_until(300, 20, 0, config_system_checks_passed, duthost) logging.info("Reload configuration") duthost.shell("sudo config reload -y &>/dev/null", executable="/bin/bash") @@ -39,7 +39,7 @@ def test_reload_configuration(duthosts, rand_one_dut_hostname, conn_graph_facts, wait_critical_processes(duthost) logging.info("Wait some time for all the transceivers to be detected") - assert wait_until(300, 20, check_all_interface_information, duthost, interfaces, xcvr_skip_list), \ + assert wait_until(300, 20, 0, check_all_interface_information, duthost, interfaces, xcvr_skip_list), \ "Not all transceivers are detected in 300 seconds" logging.info("Check transceiver status") @@ -74,7 +74,7 @@ def test_reload_configuration_checks(duthosts, rand_one_dut_hostname, localhost, out = duthost.shell("sudo config reload -y", executable="/bin/bash") # config reload command shouldn't work immediately after system reboot assert "Retry later" in out['stdout'] - assert wait_until(300, 20, config_system_checks_passed, duthost) + assert wait_until(300, 20, 0, config_system_checks_passed, duthost) # After the system checks succeed the config reload command should not throw error out = duthost.shell("sudo config reload -y", executable="/bin/bash") @@ -84,7 +84,7 @@ def test_reload_configuration_checks(duthosts, rand_one_dut_hostname, localhost, logging.info("Checking config reload after system is up") out = duthost.shell("sudo config reload -y", executable="/bin/bash") assert "Retry later" in out['stdout'] - assert wait_until(300, 20, config_system_checks_passed, duthost) + assert wait_until(300, 20, 0, config_system_checks_passed, duthost) logging.info("Stopping swss docker and checking config reload") duthost.shell("sudo service swss stop") @@ -98,4 +98,4 @@ def test_reload_configuration_checks(duthosts, rand_one_dut_hostname, localhost, out = duthost.shell("sudo config reload -y -f", executable="/bin/bash") assert "Retry later" not in out['stdout'] - assert wait_until(300, 20, config_system_checks_passed, duthost) + assert wait_until(300, 20, 0, config_system_checks_passed, duthost) diff --git a/tests/platform_tests/test_sequential_restart.py b/tests/platform_tests/test_sequential_restart.py index d5eb953f018..b549d3ad153 100644 --- a/tests/platform_tests/test_sequential_restart.py +++ b/tests/platform_tests/test_sequential_restart.py @@ -66,7 +66,7 @@ def restart_service_and_check(localhost, dut, enum_frontend_asic_index, service, wait_critical_processes(dut) logging.info("Wait some time for all the transceivers to be detected") - pytest_assert(wait_until(300, 20, check_interface_information, dut, enum_frontend_asic_index, interfaces, xcvr_skip_list), + pytest_assert(wait_until(300, 20, 0, check_interface_information, dut, enum_frontend_asic_index, interfaces, xcvr_skip_list), "Not all interface information are detected within 300 seconds") logging.info("Check transceiver status on asic %s" % enum_frontend_asic_index) diff --git a/tests/process_monitoring/test_critical_process_monitoring.py b/tests/process_monitoring/test_critical_process_monitoring.py index f210bb565da..c80b136adb6 100755 --- a/tests/process_monitoring/test_critical_process_monitoring.py +++ b/tests/process_monitoring/test_critical_process_monitoring.py @@ -167,7 +167,7 @@ def postcheck_critical_processes_status(duthost, up_bgp_neighbors): for 3 minutes. It will return False after timeout. """ logger.info("Post-checking status of critical processes and BGP sessions...") - return wait_until(POST_CHECK_THRESHOLD_SECS, POST_CHECK_INTERVAL_SECS, + return wait_until(POST_CHECK_THRESHOLD_SECS, POST_CHECK_INTERVAL_SECS, 0, post_test_check, duthost, up_bgp_neighbors) diff --git a/tests/qos/qos_sai_base.py b/tests/qos/qos_sai_base.py index 6afbc76b8d1..0ae983849d5 100644 --- a/tests/qos/qos_sai_base.py +++ b/tests/qos/qos_sai_base.py @@ -1323,7 +1323,7 @@ def is_intf_status(asic, intf, oper_state): # wait for port status to change pytest_assert( wait_until( - 10, 1, is_intf_status, frontend_asic, intf, + 10, 1, 0, is_intf_status, frontend_asic, intf, oper_state ), "Failed to update port status {} {}".format( diff --git a/tests/qos/test_buffer.py b/tests/qos/test_buffer.py index c8887055627..d0078b21688 100644 --- a/tests/qos/test_buffer.py +++ b/tests/qos/test_buffer.py @@ -410,7 +410,7 @@ def _ensure_pool_size(duthost, expected_pool_size, expected_shp_size, ingress_lo else: delay = 1 - return wait_until(timeout, delay, _ensure_pool_size, duthost, expected_pool_size, expected_shp_size, ingress_lossless_pool_oid) + return wait_until(timeout, delay, 0, _ensure_pool_size, duthost, expected_pool_size, expected_shp_size, ingress_lossless_pool_oid) def check_pg_profile(duthost, pg, expected_profile, fail_test=True): @@ -432,7 +432,7 @@ def _check_pg_profile(duthost, pg, expected_profile): profile = duthost.shell('redis-cli hget {} profile'.format(pg))['stdout'] return (profile == expected_profile) - if wait_until(10, 2, _check_pg_profile, duthost, pg, expected_profile): + if wait_until(10, 2, 0, _check_pg_profile, duthost, pg, expected_profile): return True else: if fail_test: @@ -452,7 +452,7 @@ def _check_pfc_enable(duthost, port, expected_pfc_enable_map): pfc_enable = duthost.shell('redis-cli -n 4 hget "PORT_QOS_MAP|{}" pfc_enable'.format(port))['stdout'] return (expected_pfc_enable_map == pfc_enable) - pytest_assert(wait_until(10, 2, _check_pfc_enable, duthost, port, expected_pfc_enable_map), + pytest_assert(wait_until(10, 2, 0, _check_pfc_enable, duthost, port, expected_pfc_enable_map), "Port {} pfc enable check failed expected: {} got: {}".format( port, expected_pfc_enable_map, @@ -1098,7 +1098,7 @@ def _check_buffer_profiles_for_shp(duthost, shp_enabled): # Return True only if all lossless profiles pass the check return True - pytest_assert(wait_until(20, 2, _check_buffer_profiles_for_shp, duthost, shp_enabled)) + pytest_assert(wait_until(20, 2, 0, _check_buffer_profiles_for_shp, duthost, shp_enabled)) def test_shared_headroom_pool_configure(duthosts, rand_one_dut_hostname, conn_graph_facts, port_to_test): diff --git a/tests/qos/test_qos_sai.py b/tests/qos/test_qos_sai.py index 77940867dd9..4475d457364 100644 --- a/tests/qos/test_qos_sai.py +++ b/tests/qos/test_qos_sai.py @@ -930,7 +930,7 @@ def test_qos_masic_dscp_queue_mapping( # ensure the test destination IP has a path to backend ASIC pytest_assert( wait_until( - 30, 1, self.check_v4route_backend_nhop, duthost, + 30, 1, 0, self.check_v4route_backend_nhop, duthost, test_params["src_asic"], test_params["dst_port_ip"] ), "Route {} doesn't have backend ASIC nexthop on ASIC {}".format( diff --git a/tests/read_mac/test_read_mac_metadata.py b/tests/read_mac/test_read_mac_metadata.py index 22ee6988b00..64aade175f1 100644 --- a/tests/read_mac/test_read_mac_metadata.py +++ b/tests/read_mac/test_read_mac_metadata.py @@ -82,7 +82,7 @@ def run_test_in_reinstall_loop(self): self.deploy_image_to_duthost(duthost, counter) reboot(duthost, localhost, wait=120) logger.info("Wait until system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started") + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") if current_minigraph: logger.info("Execute cli 'config load_minigraph -y' to apply new minigraph") diff --git a/tests/route/test_route_perf.py b/tests/route/test_route_perf.py index f5e91a84c21..10f4b2c63ab 100644 --- a/tests/route/test_route_perf.py +++ b/tests/route/test_route_perf.py @@ -187,7 +187,7 @@ def _check_num_routes(expected_num_routes): # Check the number of routes in ASIC_DB return count_routes(duthost) == expected_num_routes - if not wait_until(route_timeout, 0.5, _check_num_routes, expected_num_routes): + if not wait_until(route_timeout, 0.5, 0, _check_num_routes, expected_num_routes): pytest.fail('failed to add routes within time limit') # Record time when all routes show up in ASIC_DB diff --git a/tests/route/test_static_route.py b/tests/route/test_static_route.py index 9865f3ede91..669c03356f3 100644 --- a/tests/route/test_static_route.py +++ b/tests/route/test_static_route.py @@ -118,7 +118,7 @@ def generate_and_verify_traffic(duthost, ptfadapter, tbinfo, ip_dst, expected_po def wait_all_bgp_up(duthost): config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts'] bgp_neighbors = config_facts.get('BGP_NEIGHBOR', {}) - if not wait_until(300, 10, duthost.check_bgp_session_state, bgp_neighbors.keys()): + if not wait_until(300, 10, 0, duthost.check_bgp_session_state, bgp_neighbors.keys()): pytest.fail("not all bgp sessions are up after config reload") def check_route_redistribution(duthost, prefix, ipv6, removed=False): @@ -149,7 +149,7 @@ def _check_routes(): return False return True - assert(wait_until(60, 15, _check_routes)) + assert(wait_until(60, 15, 0, _check_routes)) def run_static_route_test(duthost, ptfadapter, ptfhost, tbinfo, prefix, nexthop_addrs, prefix_len, nexthop_devs, nexthop_interfaces, ipv6=False, config_reload_test=False): # Clean up arp or ndp diff --git a/tests/sai_qualify/conftest.py b/tests/sai_qualify/conftest.py index aebc6ab603e..ef10ef19211 100644 --- a/tests/sai_qualify/conftest.py +++ b/tests/sai_qualify/conftest.py @@ -159,11 +159,11 @@ def start_sai_test_conatiner_with_retry(duthost, container_name): dut_ip = duthost.host.options['inventory_manager'].get_host(duthost.hostname).vars['ansible_host'] logger.info("Checking the PRC connection before starting the {}.".format(container_name)) - rpc_ready = wait_until(1, 1, _is_rpc_server_ready, dut_ip) + rpc_ready = wait_until(1, 1, 0, _is_rpc_server_ready, dut_ip) if not rpc_ready: logger.info("Attempting to start {}.".format(container_name)) - sai_ready = wait_until(SAI_TEST_CTNR_CHECK_TIMEOUT_IN_SEC, SAI_TEST_CTNR_RESTART_INTERVAL_IN_SEC, _is_sai_test_container_restarted, duthost, container_name) + sai_ready = wait_until(SAI_TEST_CTNR_CHECK_TIMEOUT_IN_SEC, SAI_TEST_CTNR_RESTART_INTERVAL_IN_SEC, 0, _is_sai_test_container_restarted, duthost, container_name) pt_assert(sai_ready, "[{}] sai test container failed to start in {}s".format(container_name, SAI_TEST_CTNR_CHECK_TIMEOUT_IN_SEC)) logger.info("Waiting for another {} second for sai test container warm up.".format(SAI_TEST_CONTAINER_WARM_UP_IN_SEC)) time.sleep(SAI_TEST_CONTAINER_WARM_UP_IN_SEC) @@ -199,7 +199,7 @@ def _is_sai_test_container_restarted(duthost, container_name): logger.info("{} already exists, stop and remove it for a clear restart.".format(container_name)) stop_and_rm_sai_test_container(duthost, container_name) _start_sai_test_container(duthost, container_name) - rpc_ready = wait_until(RPC_RESTART_INTERVAL_IN_SEC, RPC_CHECK_INTERVAL_IN_SEC, _is_rpc_server_ready, dut_ip) + rpc_ready = wait_until(RPC_RESTART_INTERVAL_IN_SEC, RPC_CHECK_INTERVAL_IN_SEC, 0, _is_rpc_server_ready, dut_ip) if not rpc_ready: logger.info("Failed to start up {} for sai testing on DUT, stop it for a restart".format(container_name)) return rpc_ready @@ -437,7 +437,7 @@ def ready_for_sai_test(): return False return True - shutdown_check = wait_until(20, 4, ready_for_sai_test) + shutdown_check = wait_until(20, 4, 0, ready_for_sai_test) if running_services: format_list = ['{:>1}' for item in running_services] servers = ','.join(format_list) diff --git a/tests/sflow/test_sflow.py b/tests/sflow/test_sflow.py index 602fe8a1413..e5cccdcde01 100644 --- a/tests/sflow/test_sflow.py +++ b/tests/sflow/test_sflow.py @@ -450,7 +450,7 @@ def testRebootSflowEnable(self, sflowbase_config, config_sflow_agent, duthost, l verify_show_sflow(duthost,status='up',polling_int=80) duthost.command('sudo config save -y') reboot(duthost, localhost) - assert wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started" + assert wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started" verify_show_sflow(duthost,status='up',collector=['collector0','collector1'],polling_int=80) for intf in var['sflow_ports']: var['sflow_ports'][intf]['ifindex'] = get_ifindex(duthost,intf) @@ -475,7 +475,7 @@ def testRebootSflowDisable(self, sflowbase_config, duthost, localhost, partial_p active_collectors="[]" ) duthost.command('sudo config save -y') reboot(duthost, localhost) - assert wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started" + assert wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started" verify_show_sflow(duthost,status='down') for intf in var['sflow_ports']: var['sflow_ports'][intf]['ifindex'] = get_ifindex(duthost,intf) @@ -493,7 +493,7 @@ def testFastreboot(self, sflowbase_config, config_sflow_agent, duthost, localhos verify_show_sflow(duthost,status='up',collector=['collector0','collector1']) duthost.command('sudo config save -y') reboot(duthost, localhost,reboot_type='fast') - assert wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started" + assert wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started" verify_show_sflow(duthost,status='up',collector=['collector0','collector1']) for intf in var['sflow_ports']: var['sflow_ports'][intf]['ifindex'] = get_ifindex(duthost,intf) @@ -511,7 +511,7 @@ def testWarmreboot(self, sflowbase_config, duthost, localhost, partial_ptf_runne verify_show_sflow(duthost,status='up',collector=['collector0','collector1']) duthost.command('sudo config save -y') reboot(duthost, localhost,reboot_type='warm') - assert wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started" + assert wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started" verify_show_sflow(duthost,status='up',collector=['collector0','collector1']) for intf in var['sflow_ports']: var['sflow_ports'][intf]['ifindex'] = get_ifindex(duthost,intf) diff --git a/tests/show_techsupport/test_techsupport.py b/tests/show_techsupport/test_techsupport.py index e2998843cf1..8716891fc22 100644 --- a/tests/show_techsupport/test_techsupport.py +++ b/tests/show_techsupport/test_techsupport.py @@ -287,7 +287,7 @@ def test_techsupport(request, config, duthosts, enum_rand_one_per_hwsku_frontend for i in range(loop_range): logger.debug("Running show techsupport ... ") - wait_until(300, 20, execute_command, duthost, str(since)) + wait_until(300, 20, 0, execute_command, duthost, str(since)) tar_file = [j for j in pytest.tar_stdout.split('\n') if j != ''][-1] stdout = duthost.command("rm -rf {}".format(tar_file)) logger.debug("Sleeping for {} seconds".format(loop_delay)) diff --git a/tests/snappi/pfc/test_pfc_pause_lossless_with_snappi.py b/tests/snappi/pfc/test_pfc_pause_lossless_with_snappi.py index e3df1b82faf..deb7921e2c5 100644 --- a/tests/snappi/pfc/test_pfc_pause_lossless_with_snappi.py +++ b/tests/snappi/pfc/test_pfc_pause_lossless_with_snappi.py @@ -178,7 +178,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=snappi_api, @@ -243,7 +243,7 @@ def test_pfc_pause_multi_lossless_prio_reboot(snappi_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=snappi_api, diff --git a/tests/snappi/pfc/test_pfc_pause_lossy_with_snappi.py b/tests/snappi/pfc/test_pfc_pause_lossy_with_snappi.py index 4930eefe809..c33788391d4 100644 --- a/tests/snappi/pfc/test_pfc_pause_lossy_with_snappi.py +++ b/tests/snappi/pfc/test_pfc_pause_lossy_with_snappi.py @@ -179,7 +179,7 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=snappi_api, @@ -244,7 +244,7 @@ def test_pfc_pause_multi_lossy_prio_reboot(snappi_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfc_test(api=snappi_api, diff --git a/tests/snappi/pfcwd/test_pfcwd_basic_with_snappi.py b/tests/snappi/pfcwd/test_pfcwd_basic_with_snappi.py index bca2201004a..f265e8a0bd4 100644 --- a/tests/snappi/pfcwd/test_pfcwd_basic_with_snappi.py +++ b/tests/snappi/pfcwd/test_pfcwd_basic_with_snappi.py @@ -165,7 +165,7 @@ def test_pfcwd_basic_single_lossless_prio_reboot(snappi_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfcwd_basic_test(api=snappi_api, @@ -227,7 +227,7 @@ def test_pfcwd_basic_multi_lossless_prio_reboot(snappi_api, logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname)) reboot(duthost, localhost, reboot_type=reboot_type) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfcwd_basic_test(api=snappi_api, @@ -289,7 +289,7 @@ def test_pfcwd_basic_single_lossless_prio_service_restart(snappi_api, duthost.command("systemctl reset-failed {}".format(restart_service)) duthost.command("systemctl restart {}".format(restart_service)) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfcwd_basic_test(api=snappi_api, @@ -350,7 +350,7 @@ def test_pfcwd_basic_multi_lossless_prio_restart_service(snappi_api, duthost.command("systemctl reset-failed {}".format(restart_service)) duthost.command("systemctl restart {}".format(restart_service)) logger.info("Wait until the system is stable") - pytest_assert(wait_until(300, 20, duthost.critical_services_fully_started), + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started") run_pfcwd_basic_test(api=snappi_api, diff --git a/tests/snmp/conftest.py b/tests/snmp/conftest.py index 3877404cc47..ec6909c81fa 100644 --- a/tests/snmp/conftest.py +++ b/tests/snmp/conftest.py @@ -4,7 +4,7 @@ @pytest.fixture(scope="module", autouse=True) def setup_check_snmp_ready(duthosts): for duthost in duthosts: - assert wait_until(300, 20, duthost.is_service_fully_started, "snmp"), "SNMP service is not running" + assert wait_until(300, 20, 0, duthost.is_service_fully_started, "snmp"), "SNMP service is not running" def pytest_addoption(parser): """ diff --git a/tests/snmp/test_snmp_phy_entity.py b/tests/snmp/test_snmp_phy_entity.py index 624165341a6..d5906365710 100644 --- a/tests/snmp/test_snmp_phy_entity.py +++ b/tests/snmp/test_snmp_phy_entity.py @@ -613,9 +613,9 @@ def test_turn_off_psu_and_check_psu_info(duthosts, enum_rand_one_per_hwsku_hostn # turn off the first PSU first_outlet = outlet_status[0] pdu_controller.turn_off_outlet(first_outlet) - assert wait_until(30, 5, check_outlet_status, pdu_controller, first_outlet, False) + assert wait_until(30, 5, 0, check_outlet_status, pdu_controller, first_outlet, False) # wait for psud update the database - assert wait_until(180, 20, _check_psu_status_after_power_off, duthost, localhost, creds_all_duts) + assert wait_until(180, 20, 0, _check_psu_status_after_power_off, duthost, localhost, creds_all_duts) def _check_psu_status_after_power_off(duthost, localhost, creds_all_duts): diff --git a/tests/sub_port_interfaces/conftest.py b/tests/sub_port_interfaces/conftest.py index 33a4671867e..95ce4faf529 100644 --- a/tests/sub_port_interfaces/conftest.py +++ b/tests/sub_port_interfaces/conftest.py @@ -171,7 +171,7 @@ def apply_config_on_the_dut(define_sub_ports_configuration, duthost, reload_dut_ duthost.copy(content=config_template.render(sub_ports_vars), dest=sub_ports_config_path) duthost.command('sonic-cfggen -j {} --write-to-db'.format(sub_ports_config_path)) - py_assert(wait_until(3, 1, check_sub_port, duthost, sub_ports_vars['sub_ports'].keys()), + py_assert(wait_until(3, 1, 0, check_sub_port, duthost, sub_ports_vars['sub_ports'].keys()), "Some sub-ports were not created") yield sub_ports_vars diff --git a/tests/sub_port_interfaces/sub_ports_helpers.py b/tests/sub_port_interfaces/sub_ports_helpers.py index db9ea0e506c..7eb3aaca5bc 100644 --- a/tests/sub_port_interfaces/sub_ports_helpers.py +++ b/tests/sub_port_interfaces/sub_ports_helpers.py @@ -300,7 +300,7 @@ def shutdown_port(duthost, interface): interface: Interface of DUT """ duthost.shutdown(interface) - pytest_assert(wait_until(3, 1, __check_interface_state, duthost, interface, 'down'), + pytest_assert(wait_until(3, 1, 0, __check_interface_state, duthost, interface, 'down'), "DUT's port {} didn't go down as expected".format(interface)) @@ -313,7 +313,7 @@ def startup_port(duthost, interface): interface: Interface of DUT """ duthost.no_shutdown(interface) - pytest_assert(wait_until(3, 1, __check_interface_state, duthost, interface), + pytest_assert(wait_until(3, 1, 0, __check_interface_state, duthost, interface), "DUT's port {} didn't go up as expected".format(interface)) @@ -347,7 +347,7 @@ def setup_vlan(duthost, vlan_id): """ duthost.shell('config vlan add %s' % vlan_id) - pytest_assert(wait_until(3, 1, __check_vlan, duthost, vlan_id), + pytest_assert(wait_until(3, 1, 0, __check_vlan, duthost, vlan_id), "VLAN RIF Vlan{} didn't create as expected".format(vlan_id)) @@ -404,7 +404,7 @@ def remove_vlan(duthost, vlan_id): """ duthost.shell('config vlan del {}'.format(vlan_id)) - pytest_assert(wait_until(3, 1, __check_vlan, duthost, vlan_id, True), + pytest_assert(wait_until(3, 1, 0, __check_vlan, duthost, vlan_id, True), "VLAN RIF Vlan{} didn't remove as expected".format(vlan_id)) @@ -419,7 +419,7 @@ def remove_member_from_vlan(duthost, vlan_id, vlan_member): """ if __check_vlan_member(duthost, vlan_id, vlan_member): duthost.shell('config vlan member del {} {}'.format(vlan_id, vlan_member)) - pytest_assert(wait_until(3, 1, __check_vlan_member, duthost, vlan_id, vlan_member, True), + pytest_assert(wait_until(3, 1, 0, __check_vlan_member, duthost, vlan_id, vlan_member, True), "VLAN RIF Vlan{} have {} member".format(vlan_id, vlan_member)) @@ -830,7 +830,7 @@ def add_member_to_vlan(duthost, vlan_id, vlan_member): """ if not __check_vlan_member(duthost, vlan_id, vlan_member): duthost.shell('config vlan member add {} {}'.format(vlan_id, vlan_member)) - pytest_assert(wait_until(3, 1, __check_vlan_member, duthost, vlan_id, vlan_member), + pytest_assert(wait_until(3, 1, 0, __check_vlan_member, duthost, vlan_id, vlan_member), "VLAN RIF Vlan{} doesn't have {} member".format(vlan_id, vlan_member)) diff --git a/tests/sub_port_interfaces/test_show_subinterface.py b/tests/sub_port_interfaces/test_show_subinterface.py index f0ac79e129f..70bea0c7e94 100644 --- a/tests/sub_port_interfaces/test_show_subinterface.py +++ b/tests/sub_port_interfaces/test_show_subinterface.py @@ -69,7 +69,7 @@ def _remove_subintf(subintf_config): # creation verification success_show_sub_status = [] - if not wait_until(20, 5, _verify_subintf_creation, subintf_expected_config, success_show_sub_status): + if not wait_until(20, 5, 0, _verify_subintf_creation, subintf_expected_config, success_show_sub_status): pytest.fail("Failed to create subinterfaces") show_sub_status = success_show_sub_status[0] @@ -98,7 +98,7 @@ def _remove_subintf(subintf_config): # deletion verification _remove_subintf(subintf_expected_config) - if not wait_until(20, 5, _verify_subintf_removal, subintf_expected_config): + if not wait_until(20, 5, 0, _verify_subintf_removal, subintf_expected_config): pytest.fail("Failed to remove subinterfaces") show_ip_interfaces = {_["interface"]: _ for _ in duthost.show_and_parse("show ip interface") diff --git a/tests/system_health/test_system_health.py b/tests/system_health/test_system_health.py index 9ad6fba2720..44903fbed13 100644 --- a/tests/system_health/test_system_health.py +++ b/tests/system_health/test_system_health.py @@ -281,7 +281,7 @@ def test_system_health_config(duthosts, enum_rand_one_per_hwsku_hostname, device def wait_system_health_boot_up(duthost): boot_timeout = get_system_health_config(duthost, 'boot_timeout', DEFAULT_BOOT_TIMEOUT) - assert wait_until(boot_timeout, 10, redis_table_exists, duthost, STATE_DB, HEALTH_TABLE_NAME), \ + assert wait_until(boot_timeout, 10, 0, redis_table_exists, duthost, STATE_DB, HEALTH_TABLE_NAME), \ 'System health service is not working' diff --git a/tests/tacacs/test_ro_disk.py b/tests/tacacs/test_ro_disk.py index 634d1ccfe67..e56fd7dfbbe 100644 --- a/tests/tacacs/test_ro_disk.py +++ b/tests/tacacs/test_ro_disk.py @@ -29,7 +29,7 @@ def check_disk_ro(duthost): def simulate_ro(duthost): duthost.shell("echo u > /proc/sysrq-trigger") logger.info("Disk turned to RO state; pause for 30s before attempting to ssh") - assert wait_until(30, 2, check_disk_ro, duthost), "disk not in ro state" + assert wait_until(30, 2, 0, check_disk_ro, duthost), "disk not in ro state" def chk_ssh_remote_run(localhost, remote_ip, username, password, cmd): @@ -99,14 +99,14 @@ def test_ro_disk(localhost, duthosts, enum_rand_one_per_hwsku_hostname, creds_al logger.debug("user={}".format(ro_user)) - assert wait_until(600, 20, chk_ssh_remote_run, localhost, dutip, + assert wait_until(600, 20, 0, chk_ssh_remote_run, localhost, dutip, ro_user, ro_pass, "cat /etc/passwd"), "Failed to ssh as ro user" finally: logger.debug("START: reboot {} to restore disk RW state". format(enum_rand_one_per_hwsku_hostname)) do_reboot(duthost, localhost, dutip, rw_user, rw_pass) - assert wait_until(600, 20, duthost.critical_services_fully_started), "Not all critical services are fully started" + assert wait_until(600, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started" logger.debug(" END: reboot {} to restore disk RW state". format(enum_rand_one_per_hwsku_hostname)) diff --git a/tests/tacacs/utils.py b/tests/tacacs/utils.py index 0c8b695bab3..e8b79fb15a8 100644 --- a/tests/tacacs/utils.py +++ b/tests/tacacs/utils.py @@ -60,7 +60,7 @@ def setup_tacacs_server(ptfhost, creds_all_duts, duthost): # FIXME: This is a short term mitigation, we need to figure out why the tacacs+ server does not start # reliably all of a sudden. - wait_until(5, 1, start_tacacs_server, ptfhost) + wait_until(5, 1, 0, start_tacacs_server, ptfhost) check_all_services_status(ptfhost) diff --git a/tests/telemetry/test_telemetry.py b/tests/telemetry/test_telemetry.py index d0e7344c300..d5e2b2fc54e 100644 --- a/tests/telemetry/test_telemetry.py +++ b/tests/telemetry/test_telemetry.py @@ -93,7 +93,7 @@ def setup_streaming_telemetry(duthosts, rand_one_dut_hostname, localhost, ptfho setup_telemetry_forpyclient(duthost) # Wait until telemetry was restarted - pytest_assert(wait_until(100, 10, duthost.is_service_fully_started, "telemetry"), "TELEMETRY not started.") + pytest_assert(wait_until(100, 10, 0, duthost.is_service_fully_started, "telemetry"), "TELEMETRY not started.") logger.info("telemetry process restarted. Now run pyclient on ptfdocker") # Wait until the TCP port was opened diff --git a/tests/test_pretest.py b/tests/test_pretest.py index 54623a1a6d9..faa85a32b2a 100644 --- a/tests/test_pretest.py +++ b/tests/test_pretest.py @@ -41,13 +41,13 @@ def test_features_state(duthosts, enum_dut_hostname, localhost): """ duthost = duthosts[enum_dut_hostname] logger.info("Checking the state of each feature in 'CONFIG_DB' ...") - if not wait_until(180, FEATURE_STATE_VERIFYING_INTERVAL_SECS, verify_features_state, duthost): + if not wait_until(180, FEATURE_STATE_VERIFYING_INTERVAL_SECS, 0, verify_features_state, duthost): logger.warn("Not all states of features in 'CONFIG_DB' are valid, rebooting DUT {}".format(duthost.hostname)) reboot(duthost, localhost) # Some services are not ready immeidately after reboot wait_critical_processes(duthost) - pytest_assert(wait_until(FEATURE_STATE_VERIFYING_THRESHOLD_SECS, FEATURE_STATE_VERIFYING_INTERVAL_SECS, + pytest_assert(wait_until(FEATURE_STATE_VERIFYING_THRESHOLD_SECS, FEATURE_STATE_VERIFYING_INTERVAL_SECS, 0, verify_features_state, duthost), "Not all service states are valid!") logger.info("The states of features in 'CONFIG_DB' are all valid.") diff --git a/tests/voq/test_voq_ipfwd.py b/tests/voq/test_voq_ipfwd.py index 2f3e362943c..9f59aa6e059 100644 --- a/tests/voq/test_voq_ipfwd.py +++ b/tests/voq/test_voq_ipfwd.py @@ -983,7 +983,7 @@ def test_front_panel_linkflap_port(self, duthosts, all_cfg_facts, self.linkflap_up(fanout, fanport, ports['portA']['dut'], lport) # need bgp to establish - wait_until(200, 20, bgp_established, ports['portA']['dut'], ports['portA']['asic']) + wait_until(200, 20, 0, bgp_established, ports['portA']['dut'], ports['portA']['asic']) # Validate from port A and neighbor A that everything is good after port is up. logger.info("=" * 80) diff --git a/tests/voq/test_voq_nbr.py b/tests/voq/test_voq_nbr.py index 26b202eac50..fe9bacdee7d 100644 --- a/tests/voq/test_voq_nbr.py +++ b/tests/voq/test_voq_nbr.py @@ -744,10 +744,10 @@ def test_neighbor_hw_mac_change(duthosts, enum_rand_one_per_hwsku_frontend_hostn # Check neighbor on local linecard logger.info("*" * 60) logger.info("Verify initial neighbor: %s, port %s", neighbor, local_port) - pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False), "MAC {} didn't change in ARP table".format(original_mac)) sonic_ping(asic, neighbor, verbose=True) - pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, per_host, asic, neighbor, original_mac), "MAC {} didn't change in ARP table".format(original_mac)) dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs) @@ -761,11 +761,11 @@ def test_neighbor_hw_mac_change(duthosts, enum_rand_one_per_hwsku_frontend_hostn if ":" in neighbor: logger.info("Force neighbor solicitation to workaround long IPV6 timer.") asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port)) - pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC, checkstate=False), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, per_host, asic, neighbor, NEW_MAC, checkstate=False), "MAC {} didn't change in ARP table".format(NEW_MAC)) sonic_ping(asic, neighbor, verbose=True) - pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, NEW_MAC), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, per_host, asic, neighbor, NEW_MAC), "MAC {} didn't change in ARP table".format(NEW_MAC)) logger.info("Verify neighbor after mac change: %s, port %s", neighbor, local_port) check_one_neighbor_present(duthosts, per_host, asic, neighbor, nbrhosts, all_cfg_facts) @@ -783,10 +783,10 @@ def test_neighbor_hw_mac_change(duthosts, enum_rand_one_per_hwsku_frontend_hostn logger.info("Force neighbor solicitation to workaround long IPV6 timer.") asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port)) pytest_assert( - wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False), + wait_until(60, 2, 0, check_arptable_mac, per_host, asic, neighbor, original_mac, checkstate=False), "MAC {} didn't change in ARP table".format(original_mac)) sonic_ping(asic, neighbor, verbose=True) - pytest_assert(wait_until(60, 2, check_arptable_mac, per_host, asic, neighbor, original_mac), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, per_host, asic, neighbor, original_mac), "MAC {} didn't change in ARP table".format(original_mac)) dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, nbr_to_test, nbrhosts, all_cfg_facts, nbr_macs) @@ -829,7 +829,7 @@ def linkflap_down(self, fanout, fanport, dut, dut_intf): """ logger.info("Bring down link: %s/%s <-> %s/%s", fanout.hostname, fanport, dut.hostname, dut_intf) fanout.shutdown(fanport) - pytest_assert(wait_until(30, 1, self.check_intf_status, dut, dut_intf, 'down'), + pytest_assert(wait_until(30, 1, 0, self.check_intf_status, dut, dut_intf, 'down'), "dut port {} didn't go down as expected".format(dut_intf)) def linkflap_up(self, fanout, fanport, dut, dut_intf): @@ -848,7 +848,7 @@ def linkflap_up(self, fanout, fanport, dut, dut_intf): """ logger.info("Bring up link: %s/%s <-> %s/%s", fanout.hostname, fanport, dut.hostname, dut_intf) fanout.no_shutdown(fanport) - pytest_assert(wait_until(60, 1, self.check_intf_status, dut, dut_intf, 'up'), + pytest_assert(wait_until(60, 1, 0, self.check_intf_status, dut, dut_intf, 'up'), "dut port {} didn't go up as expected".format(dut_intf)) def localport_admindown(self, dut, asic, dut_intf): @@ -865,7 +865,7 @@ def localport_admindown(self, dut, asic, dut_intf): """ logger.info("Admin down port %s/%s", dut.hostname, dut_intf) asic.shutdown_interface(dut_intf) - pytest_assert(wait_until(30, 1, self.check_intf_status, dut, dut_intf, 'down'), + pytest_assert(wait_until(30, 1, 0, self.check_intf_status, dut, dut_intf, 'down'), "dut port {} didn't go down as expected".format(dut_intf)) def localport_adminup(self, dut, asic, dut_intf): @@ -882,7 +882,7 @@ def localport_adminup(self, dut, asic, dut_intf): """ logger.info("Admin up port %s/%s", dut.hostname, dut_intf) asic.startup_interface(dut_intf) - pytest_assert(wait_until(30, 1, self.check_intf_status, dut, dut_intf, 'up'), + pytest_assert(wait_until(30, 1, 0, self.check_intf_status, dut, dut_intf, 'up'), "dut port {} didn't go up as expected".format(dut_intf)) @@ -978,7 +978,7 @@ def test_front_panel_admindown_port(self, duthosts, enum_rand_one_per_hwsku_fron sonic_ping(asic, neighbor) for neighbor in neighbors: - pytest_assert(wait_until(60, 2, check_arptable_state, per_host, asic, neighbor, "REACHABLE"), + pytest_assert(wait_until(60, 2, 0, check_arptable_state, per_host, asic, neighbor, "REACHABLE"), "STATE for neighbor {} did not change to reachable".format(neighbor)) dump_and_verify_neighbors_on_asic(duthosts, per_host, asic, neighbors, nbrhosts, all_cfg_facts, nbr_macs) @@ -1167,7 +1167,7 @@ def test_gratarp_macchange(self, duthosts, enum_rand_one_per_hwsku_frontend_host logger.info("%s port %s is on ptf port: %s", duthost.hostname, local_port, tb_port) logger.info("-" * 60) sonic_ping(asic, neighbor) - pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, duthost, asic, neighbor, original_mac), "MAC {} didn't change in ARP table".format(original_mac)) check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts) @@ -1176,13 +1176,13 @@ def test_gratarp_macchange(self, duthosts, enum_rand_one_per_hwsku_frontend_host change_mac(nbrhosts[nbrinfo['vm']], nbrinfo['shell_intf'], NEW_MAC) self.send_grat_pkt(NEW_MAC, neighbor, int(tb_port)) - pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=False), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=False), "MAC {} didn't change in ARP table of neighbor {}".format(NEW_MAC, neighbor)) try: sonic_ping(asic, neighbor) except AssertionError: logging.info("No initial response from ping, begin poll to see if ARP table responds.") - pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=True), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, duthost, asic, neighbor, NEW_MAC, checkstate=True), "MAC {} didn't change in ARP table of neighbor {}".format(NEW_MAC, neighbor)) check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts) ping_all_neighbors(duthosts, all_cfg_facts, [neighbor]) @@ -1195,10 +1195,10 @@ def test_gratarp_macchange(self, duthosts, enum_rand_one_per_hwsku_frontend_host logger.info("Force neighbor solicitation to workaround long IPV6 timer.") asic_cmd(asic, "ndisc6 %s %s" % (neighbor, local_port)) pytest_assert( - wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac, checkstate=False), + wait_until(60, 2, 0, check_arptable_mac, duthost, asic, neighbor, original_mac, checkstate=False), "MAC {} didn't change in ARP table".format(original_mac)) sonic_ping(asic, neighbor, verbose=True) - pytest_assert(wait_until(60, 2, check_arptable_mac, duthost, asic, neighbor, original_mac), + pytest_assert(wait_until(60, 2, 0, check_arptable_mac, duthost, asic, neighbor, original_mac), "MAC {} didn't change in ARP table".format(original_mac)) check_one_neighbor_present(duthosts, duthost, asic, neighbor, nbrhosts, all_cfg_facts) diff --git a/tests/vrf/test_vrf.py b/tests/vrf/test_vrf.py index 9ca749cf070..ad183a68d97 100644 --- a/tests/vrf/test_vrf.py +++ b/tests/vrf/test_vrf.py @@ -1047,11 +1047,11 @@ def test_vrf_swss_warm_reboot(self, duthosts, rand_one_dut_hostname, cfg_facts, "Some components didn't finish reconcile: {} ...".format(tbd_comp_list) # basic check after warm reboot - assert wait_until(300, 20, duthost.critical_services_fully_started), \ + assert wait_until(300, 20, 0, duthost.critical_services_fully_started), \ "All critical services should fully started!{}".format(duthost.critical_services) up_ports = [p for p, v in cfg_facts['PORT'].items() if v.get('admin_status', None) == 'up' ] - assert wait_until(300, 20, check_interface_status, duthost, up_ports), \ + assert wait_until(300, 20, 0, check_interface_status, duthost, up_ports), \ "All interfaces should be up!" def test_vrf_system_warm_reboot(self, duthosts, rand_one_dut_hostname, localhost, cfg_facts, partial_ptf_runner): @@ -1090,10 +1090,10 @@ def test_vrf_system_warm_reboot(self, duthosts, rand_one_dut_hostname, localhost assert len(tbd_comp_list) == 0, "Some components didn't finish reconcile: {} ...".format(tbd_comp_list) # basic check after warm reboot - assert wait_until(300, 20, duthost.critical_services_fully_started), "Not all critical services are fully started" + assert wait_until(300, 20, 0, duthost.critical_services_fully_started), "Not all critical services are fully started" up_ports = [p for p, v in cfg_facts['PORT'].items() if v.get('admin_status', None) == 'up' ] - assert wait_until(300, 20, check_interface_status, duthost, up_ports), "Not all interfaces are up" + assert wait_until(300, 20, 0, check_interface_status, duthost, up_ports), "Not all interfaces are up" class TestVrfCapacity(): @@ -1331,7 +1331,7 @@ def setup_vrf_unbindintf(self, duthosts, rand_one_dut_hostname, ptfhost, tbinfo, # -------- Teardown ---------- if self.c_vars['rebind_intf']: self.rebind_intf(duthost) - wait_until(120, 10, check_bgp_facts, duthost, cfg_facts) + wait_until(120, 10, 0, check_bgp_facts, duthost, cfg_facts) def rebind_intf(self, duthost): duthost.shell("config interface vrf bind PortChannel0001 Vrf1") @@ -1346,7 +1346,7 @@ def setup_vrf_rebind_intf(self, duthosts, rand_one_dut_hostname, cfg_facts): self.c_vars['rebind_intf'] = False # Mark to skip rebind interface during teardown # check bgp session state after rebind - assert wait_until(120, 10, check_bgp_facts, duthost, cfg_facts), \ + assert wait_until(120, 10, 0, check_bgp_facts, duthost, cfg_facts), \ "Bgp sessions should be re-estabalished after Portchannel0001 rebind to Vrf" def test_pc1_ip_addr_flushed(self, duthosts, rand_one_dut_hostname): @@ -1484,7 +1484,7 @@ def setup_vrf_deletion(self, duthosts, rand_one_dut_hostname, ptfhost, tbinfo, c # -------- Teardown ---------- if self.c_vars['restore_vrf']: self.restore_vrf(duthost) - wait_until(120, 10, check_bgp_facts, duthost, cfg_facts) + wait_until(120, 10, 0, check_bgp_facts, duthost, cfg_facts) @pytest.fixture(scope='class') def setup_vrf_restore(self, duthosts, rand_one_dut_hostname, cfg_facts): @@ -1493,7 +1493,7 @@ def setup_vrf_restore(self, duthosts, rand_one_dut_hostname, cfg_facts): self.c_vars['restore_vrf'] = False # Mark to skip restore vrf during teardown # check bgp session state after restore - assert wait_until(120, 10, check_bgp_facts, duthost, cfg_facts), \ + assert wait_until(120, 10, 0, check_bgp_facts, duthost, cfg_facts), \ "Bgp sessions should be re-estabalished after restore Vrf1" def test_pc1_ip_addr_flushed(self, duthosts, rand_one_dut_hostname): diff --git a/tests/vxlan/test_vnet_route_leak.py b/tests/vxlan/test_vnet_route_leak.py index d72ebaace83..c4bf63d877e 100644 --- a/tests/vxlan/test_vnet_route_leak.py +++ b/tests/vxlan/test_vnet_route_leak.py @@ -73,7 +73,7 @@ def configure_dut(minigraph_facts, duthosts, rand_one_dut_hostname, vnet_config, logger.info("Restarting BGP and waiting for BGP sessions") duthost.shell(RESTART_BGP_CMD) - if not wait_until(BGP_WAIT_TIMEOUT, BGP_POLL_RATE, bgp_connected, duthost): + if not wait_until(BGP_WAIT_TIMEOUT, BGP_POLL_RATE, 0, bgp_connected, duthost): logger.warning("BGP sessions not up {} seconds after BGP restart, restoring with `config_reload`".format(BGP_WAIT_TIMEOUT)) config_reload(duthost) else: @@ -189,7 +189,7 @@ def test_vnet_route_leak(configure_dut, duthosts, rand_one_dut_hostname): logger.info("Restarting BGP") duthost.shell(RESTART_BGP_CMD) - pytest_assert(wait_until(BGP_WAIT_TIMEOUT, BGP_POLL_RATE, bgp_connected, duthost), BGP_ERROR_TEMPLATE.format(BGP_WAIT_TIMEOUT)) + pytest_assert(wait_until(BGP_WAIT_TIMEOUT, BGP_POLL_RATE, 0, bgp_connected, duthost), BGP_ERROR_TEMPLATE.format(BGP_WAIT_TIMEOUT)) leaked_routes = get_leaked_routes(duthost) pytest_assert(not leaked_routes, LEAKED_ROUTES_TEMPLATE.format(leaked_routes)) @@ -198,7 +198,7 @@ def test_vnet_route_leak(configure_dut, duthosts, rand_one_dut_hostname): duthost.shell(CONFIG_SAVE_CMD) config_reload(duthost) - pytest_assert(wait_until(BGP_WAIT_TIMEOUT, BGP_POLL_RATE, bgp_connected, duthost), BGP_ERROR_TEMPLATE.format(BGP_WAIT_TIMEOUT)) + pytest_assert(wait_until(BGP_WAIT_TIMEOUT, BGP_POLL_RATE, 0, bgp_connected, duthost), BGP_ERROR_TEMPLATE.format(BGP_WAIT_TIMEOUT)) leaked_routes = get_leaked_routes(duthost) pytest_assert(not leaked_routes, LEAKED_ROUTES_TEMPLATE.format(leaked_routes))