Stabilize test_reload_dpu.py (sonic-net#21782)

Enhancement is for the switch memory exhaustion and kernel panic test
cases.

1. Increase DUT_ABSENT_TIMEOUT_FOR_MEMORY_EXHAUSTION as it takes time to
   exhaust the memory.
2. Fail the test if the wait_for ssh shutdown fails.
3. Use a shorter retry delay in the common method
   ssh_connection_with_retry. The check happens after the delay, if the
delay is bigger than the timeout, this wait_for always fails. See
ansible document:
https://docs.ansible.com/projects/ansible/latest/collections/ansible/builtin/wait_for_module.html.

Signed-off-by: Cong Hou <congh@nvidia.com>
Signed-off-by: Zhuohui Tan <zhuohui.tan@amd.com>

Author: congh-nvidia

tests/common/reboot.py

@@ -736,7 +736,8 @@ def ssh_connection_with_retry(localhost, host_ip, port, delay, timeout):
         'search_regex': SONIC_SSH_REGEX
     }
     short_timeout = 40
-    params_to_update_list = [{}, {'search_regex': None, 'timeout': short_timeout}]
+    short_delay = 10
+    params_to_update_list = [{}, {'search_regex': None, 'timeout': short_timeout, 'delay': short_delay}]
     for num_try, params_to_update in enumerate(params_to_update_list):
         iter_connection_params = default_connection_params.copy()
         iter_connection_params.update(params_to_update)

tests/smartswitch/platform_tests/test_reload_dpu.py

@@ -24,7 +24,7 @@
 kernel_panic_cmd = "sudo nohup bash -c 'sleep 5 && echo c > /proc/sysrq-trigger' &"
 memory_exhaustion_cmd = "sudo nohup bash -c 'sleep 5 && tail /dev/zero' &"
 DUT_ABSENT_TIMEOUT_FOR_KERNEL_PANIC = 100
-DUT_ABSENT_TIMEOUT_FOR_MEMORY_EXHAUSTION = 100
+DUT_ABSENT_TIMEOUT_FOR_MEMORY_EXHAUSTION = 240
 MAX_COOL_OFF_TIME = 300
 
 
@@ -121,8 +121,7 @@ def test_dpu_status_post_switch_mem_exhaustion(duthosts, dpuhosts,
                        state='absent',
                        search_regex=SONIC_SSH_REGEX,
                        delay=10,
-                       timeout=DUT_ABSENT_TIMEOUT_FOR_MEMORY_EXHAUSTION,
-                       module_ignore_errors=True)
+                       timeout=DUT_ABSENT_TIMEOUT_FOR_MEMORY_EXHAUSTION)
 
     logging.info("Executing post test check")
     post_test_switch_check(duthost, localhost,
@@ -164,8 +163,7 @@ def test_dpu_status_post_switch_kernel_panic(duthosts, dpuhosts,
                        state='absent',
                        search_regex=SONIC_SSH_REGEX,
                        delay=10,
-                       timeout=DUT_ABSENT_TIMEOUT_FOR_KERNEL_PANIC,
-                       module_ignore_errors=True)
+                       timeout=DUT_ABSENT_TIMEOUT_FOR_KERNEL_PANIC)
 
     logging.info("Executing post test check")
     post_test_switch_check(duthost, localhost,