[dualtor] Add test to simulate server reboot

tests/common/dualtor/icmp_responder_control.py

@@ -38,3 +38,40 @@ def _pause_icmp_respond(mux_ports):
     yield _pause_icmp_respond
 
     ptfhost.shell("supervisorctl restart icmp_responder", module_ignore_errors=True)
+
+
+def set_supervisorctl_status_icmp_responder(ptfhost, cmd, status):
+
+    icmp_responder_status = ptfhost.shell("supervisorctl status icmp_responder",
+                                          module_ignore_errors=True)["stdout"]
+    if status in icmp_responder_status:
+        raise RuntimeError(f"icmp_responder is already in {status} state")
+
+    ptfhost.shell(f'supervisorctl {cmd} icmp_responder', module_ignore_errors=True)
+
+    icmp_responder_status = ptfhost.shell("supervisorctl status icmp_responder",
+                                          module_ignore_errors=True)["stdout"]
+    if status not in icmp_responder_status:
+        raise RuntimeError(f"could not set icmp_responder to {status} state")
+
+
+@pytest.fixture
+def shutdown_icmp_responder(ptfhost):    # noqa F811
+
+    def _shutdown_icmp_responder():
+        cmd = 'stop'
+        status = 'STOPPED'
+        set_supervisorctl_status_icmp_responder(ptfhost, cmd, status)
+
+    yield _shutdown_icmp_responder
+
+
+@pytest.fixture
+def start_icmp_responder(ptfhost):    # noqa F811
+
+    def _start_icmp_responder():
+        cmd = 'start'
+        status = 'RUNNING'
+        set_supervisorctl_status_icmp_responder(ptfhost, cmd, status)
+
+    yield _start_icmp_responder

tests/dualtor_mgmt/test_server_failure.py

@@ -17,6 +17,8 @@
 from tests.common.fixtures.ptfhost_utils import change_mac_addresses, run_garp_service, \
                                                 run_icmp_responder                                      # noqa: F401
 from tests.common.utilities import wait_until
+from tests.common.dualtor.icmp_responder_control import shutdown_icmp_responder                         # noqa: F401
+from tests.common.dualtor.icmp_responder_control import start_icmp_responder                            # noqa: F401
 
 
 pytestmark = [
@@ -90,3 +92,42 @@ def lower_tor_mux_state_verfication(state, health):
                       "mux_cable status is unexpected. Should be (standby, unhealthy)")
         pytest_assert(wait_until(30, 1, 0, lower_tor_mux_state_verfication, 'standby', 'unhealthy'),
                       "mux_cable status is unexpected. Should be (standby, unhealthy)")
+
+
+@pytest.mark.enable_active_active
+def test_server_reboot(cable_type, start_icmp_responder, shutdown_icmp_responder,  # noqa: F811
+                       active_active_ports, upper_tor_host, lower_tor_host):       # noqa: F811
+
+    def upper_tor_mux_state_verification(state, health):
+        mux_state_upper_tor = show_muxcable_status(upper_tor_host)
+        return (mux_state_upper_tor[test_iface]['status'] == state and
+                mux_state_upper_tor[test_iface]['health'] == health)
+
+    def lower_tor_mux_state_verfication(state, health):
+        mux_state_lower_tor = show_muxcable_status(lower_tor_host)
+        return (mux_state_lower_tor[test_iface]['status'] == state and
+                mux_state_lower_tor[test_iface]['health'] == health)
+
+    if cable_type == CableType.active_active:
+        test_iface = random.choice(active_active_ports)
+
+        pytest_assert(upper_tor_mux_state_verification('active', 'healthy'),
+                      "mux_cable status is unexpected. Should be (active, healthy)")
+        pytest_assert(lower_tor_mux_state_verfication('active', 'healthy'),
+                      "mux_cable status is unexpected. Should be (active, healthy)")
+
+        # simulate reboot by stopping and starting icmp_responder
+
+        shutdown_icmp_responder()
+
+        pytest_assert(wait_until(30, 1, 0, upper_tor_mux_state_verification, 'standby', 'unhealthy'),
+                      "mux_cable status is unexpected. Should be (standby, unhealthy)")
+        pytest_assert(wait_until(30, 1, 0, lower_tor_mux_state_verfication, 'standby', 'unhealthy'),
+                      "mux_cable status is unexpected. Should be (standby, unhealthy)")
+
+        start_icmp_responder()
+
+        pytest_assert(wait_until(30, 1, 0, upper_tor_mux_state_verification, 'active', 'healthy'),
+                      "mux_cable status is unexpected. Should be (active, healthy)")
+        pytest_assert(wait_until(30, 1, 0, lower_tor_mux_state_verfication, 'active', 'healthy'),
+                      "mux_cable status is unexpected. Should be (active, healthy)")