Add test plan for reboot blocking mode (#19600)

What is the motivation for this PR?
https://github.com/sonic-net/SONiC/blob/master/doc/reboot/Reboot_BlockingMode_HLD.md

How did you do it?
https://github.com/sonic-net/SONiC/blob/master/doc/reboot/Reboot_BlockingMode_HLD.md

Signed-off-by: Litao Yu <lityu@microsoft.com>

Author: lityu

docs/testplan/reboot-blocking_mode-test-plan.md

@@ -0,0 +1,50 @@
+# Reboot Blocking Mode Test Plan
+
+## 1 Overview
+
+The purpose is to test the functionality of **Reboot Blocking Mode** feature on SONiC switch.
+
+For details of the feature design, please refer to HLD: [Reboot support BlockingMode in SONiC](https://github.com/sonic-net/SONiC/blob/master/doc/reboot/Reboot_BlockingMode_HLD.md).
+
+### 1.1 Scope
+
+The test is targeting a running SONiC system will fully functioning configuration. The purpose of this test is to verify the function of reboot BlockingMode with CLI and config file.
+
+### 1.2 Testbed
+
+The test can run on both physical and virtual testbeds with any topology.
+
+### 1.3 Limitation
+
+The blocking mode only affect device sku with no platform reboot enabled. So test will always success on that kinds of hardware sku.
+
+## 2 Setup Configuration
+
+Because in non-blocking mode, the CLI output is unpredictable. So we need to mock the original reboot file `/sbin/reboot`. We will update this file as an empty script so that in non-BlockingMode, we will always quickly complete the `reboot` command.
+
+## 3 Test
+
+### Test for BlockingMode CLI
+#### Test case #1 - Verify original logic will not block
+1. Run command `reboot; echo "ExpectedFinished"`. The command needs to have a timeout with 10mins. This is to avoid the script blocked unexpected.
+1. Check if the command output contains `ExpectedFinished` as expected.
+
+#### Test case #2 - Verify running output when blocking mode enabled
+1. Run command `reboot -b -v; echo "UnexpectedFinished"`. The command needs to have a timeout with 10mins.
+1. Check if the command output not contains `UnexpectedFinished` as expected.
+1. Check if there are extra dots after `Issuing OS-level reboot ...` output.
+
+### Test for BlockingMode config file
+#### Test case #1 - Verify timeout config for blocking mode with config file
+1. Backup the config file `/etc/sonic/reboot.conf` if exists. Update the following configs to the config file:
+   ```
+   blocking_mode=true
+   blocking_mode_timeout=0
+   show_timer=true
+   ```
+1. Run command `reboot; echo "UnexpectedFinished"`. The command needs to have a timeout with 10mins.
+1. Check if the command output not contains `UnexpectedFinished` as expected.
+1. Restore the config file `/etc/sonic/reboot.conf`
+
+## 4 Cleanup
+Since the reboot script already killed the SONiC modules, we need to do another reboot after restore `/sbin/reboot`.

tests/common/platform/processes_utils.py

@@ -63,16 +63,18 @@ def check_critical_processes(dut, watch_secs=0):
         watch_secs = watch_secs - 5
 
 
-def wait_critical_processes(dut):
+def wait_critical_processes(dut, timeout=None):
     """
     @summary: wait until all critical processes are healthy.
     @param dut: The AnsibleHost object of DUT. For interacting with DUT.
+    @param timeout: customized timeout value in seconds. If specified, it overwrites the value from inventory file.
     """
-    timeout = reset_timeout(dut)
-    # No matter what we set in inventory file, we always set sup timeout to 900
-    # because most SUPs have 10+ dockers that need to come up
-    if dut.is_supervisor_node():
-        timeout = 900
+    if timeout is None:
+        timeout = reset_timeout(dut)
+        # No matter what we set in inventory file, we always set sup timeout to 900
+        # because most SUPs have 10+ dockers that need to come up
+        if dut.is_supervisor_node():
+            timeout = 900
     logging.info("Wait until all critical processes are healthy in {} sec"
                  .format(timeout))
     pytest_assert(wait_until(timeout, 20, 0, _all_critical_processes_healthy, dut),

tests/reboot/test_reboot_blocking_mode.py

@@ -0,0 +1,165 @@
+import pytest
+import re
+from tests.common.helpers.assertions import pytest_assert
+from tests.common.platform.processes_utils import wait_critical_processes
+
+pytestmark = [
+    pytest.mark.disable_loganalyzer,
+    pytest.mark.topology('any'),
+]
+
+COMMAND_TIMEOUT = 90  # seconds
+
+
+def check_if_platform_reboot_enabled(duthost) -> bool:
+    platform = get_command_result(duthost, "sonic-cfggen -H -v DEVICE_METADATA.localhost.platform")
+    return check_if_dut_file_exist(duthost, "/usr/share/sonic/device/{}/platform_reboot".format(platform))
+
+
+def mock_systemctl_reboot(duthost):
+    if not check_if_dut_file_exist(duthost, "/sbin/reboot.bak"):
+        # Check exist to avoid override original reboot file.
+        execute_command(duthost, "sudo mv /sbin/reboot /sbin/reboot.bak")
+    execute_command(duthost, "sudo echo \"\" > /sbin/reboot")
+    execute_command(duthost, "sudo chmod +x /sbin/reboot")
+    execute_command_ignore_error(duthost, "sudo /usr/local/bin/watchdogutil disarm")
+
+    # Disable watch dog to avoid reboot too early.
+    execute_command(
+        duthost,
+        "sudo sed -i 's#/usr/local/bin/watchdogutil#/usr/local/bin/disabled_watchdogutil#g' /usr/local/bin/reboot")
+
+
+def restore_systemctl_reboot_and_reboot(duthost):
+    if not check_if_dut_file_exist(duthost, "/sbin/reboot.bak"):
+        return
+    execute_command(duthost, "sudo rm /sbin/reboot")
+    execute_command(duthost, "sudo mv /sbin/reboot.bak /sbin/reboot")
+    execute_command(
+        duthost,
+        "sudo sed -i 's#/usr/local/bin/disabled_watchdogutil#/usr/local/bin/watchdogutil#g' /usr/local/bin/reboot")
+    execute_command(duthost, "sudo reboot")
+
+    timeout = None
+    if duthost.is_supervisor_node():
+        timeout = 900
+    elif duthost.is_multi_asic:
+        timeout = 420
+    wait_critical_processes(duthost, timeout=timeout)
+
+
+def mock_reboot_config_file(duthost):
+    if (
+        check_if_dut_file_exist(duthost, "/etc/sonic/reboot.conf")
+        and not check_if_dut_file_exist(duthost, "/etc/sonic/reboot.conf.bak")
+    ):
+        execute_command(duthost, "sudo mv /etc/sonic/reboot.conf /etc/sonic/reboot.conf.bak")
+    execute_command(
+        duthost,
+        "echo -e \"blocking_mode=true\\nshow_timer=true\" > /etc/sonic/reboot.conf")
+
+
+def mock_reboot_config_file_with_0_timeout(duthost):
+    if (
+        check_if_dut_file_exist(duthost, "/etc/sonic/reboot.conf")
+        and not check_if_dut_file_exist(duthost, "/etc/sonic/reboot.conf.bak")
+    ):
+        execute_command(duthost, "sudo mv /etc/sonic/reboot.conf /etc/sonic/reboot.conf.bak")
+    execute_command(
+        duthost,
+        "echo -e \"blocking_mode=true\\nblocking_mode_timeout=0\\nshow_timer=true\" > /etc/sonic/reboot.conf")
+
+
+def restore_reboot_config_file(duthost):
+    execute_command(duthost, "sudo rm /etc/sonic/reboot.conf")
+    if check_if_dut_file_exist(duthost, "/etc/sonic/reboot.conf.bak"):
+        execute_command(duthost, "sudo mv /etc/sonic/reboot.conf.bak /etc/sonic/reboot.conf")
+
+
+def execute_command(duthost, cmd):
+    result = duthost.shell(cmd)
+    pytest_assert(result["rc"] == 0, "Unexpected rc: {}".format(result["rc"]))
+
+
+def execute_command_ignore_error(duthost, cmd):
+    duthost.shell(cmd, module_ignore_errors=True)
+
+
+def get_command_result(duthost, cmd):
+    result = duthost.shell(cmd, module_ignore_errors=True)
+    return result["stdout"]
+
+
+def check_if_dut_file_exist(duthost, filepath) -> bool:
+    result = duthost.shell(f"test -f {filepath} && echo true || echo false", module_ignore_errors=True)
+    return "true" in result["stdout"]
+
+
+class TestRebootBlockingModeCLI:
+    @pytest.fixture(autouse=True, scope="function")
+    def setup_teardown(
+        self,
+        duthosts,
+        enum_rand_one_per_hwsku_hostname
+    ):
+        duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+        if check_if_platform_reboot_enabled(duthost):
+            pytest.skip("Skip test because platform reboot is enabled.")
+
+        mock_systemctl_reboot(duthost)
+        yield
+        restore_systemctl_reboot_and_reboot(duthost)
+
+    def test_non_blocking_mode(
+        self,
+        duthosts,
+        enum_rand_one_per_hwsku_hostname
+    ):
+        duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+        result = get_command_result(
+            duthost,
+            f"sudo timeout {COMMAND_TIMEOUT}s bash -c 'sudo reboot; echo \"ExpectedFinished\"'")
+        pytest_assert("ExpectedFinished" in result, "Reboot didn't exited as expected.")
+
+    def test_blocking_mode(
+        self,
+        duthosts,
+        enum_rand_one_per_hwsku_hostname
+    ):
+        duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+        result = get_command_result(
+            duthost,
+            f"sudo timeout {COMMAND_TIMEOUT}s bash -c 'sudo reboot -b -v; echo \"UnexpectedFinished\"'")
+        pytest_assert("UnexpectedFinished" not in result, "Reboot script didn't blocked as expected.")
+        pattern = r".*\n[.]+$"
+        pytest_assert(re.search(pattern, result), "Cannot find dots as expected in output: {}".format(result))
+
+
+class TestRebootBlockingModeConfigFile:
+    @pytest.fixture(autouse=True, scope="function")
+    def setup_teardown(
+        self,
+        duthosts,
+        enum_rand_one_per_hwsku_hostname
+    ):
+        duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+        if check_if_platform_reboot_enabled(duthost):
+            pytest.skip("Skip test because platform reboot is enabled.")
+
+        mock_systemctl_reboot(duthost)
+        yield
+
+        restore_reboot_config_file(duthost)
+        restore_systemctl_reboot_and_reboot(duthost)
+
+    def test_timeout_for_blocking_mode(
+        self,
+        duthosts,
+        enum_rand_one_per_hwsku_hostname
+    ):
+        duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+        mock_reboot_config_file_with_0_timeout(duthost)
+        result = get_command_result(
+            duthost,
+            f"sudo timeout {COMMAND_TIMEOUT}s bash -c 'sudo reboot; echo \"ExpectedFinished\"'")
+        pytest_assert("ExpectedFinished" in result, "Reboot didn't exited as expected.")