[recovery] introduce adaptive recover method

tests/common/devices.py

@@ -350,6 +350,24 @@ def get_image_info(self):
     def get_asic_type(self):
         return self.facts["asic_type"]
 
+    def shutdown(self, ifname):
+        """
+            Shutdown interface specified by ifname
+
+            Args:
+                ifname: the interface to shutdown
+        """
+        return self.command("sudo config interface shutdown {}".format(ifname))
+
+    def no_shutdown(self, ifname):
+        """
+            Bring up interface specified by ifname
+
+            Args:
+                ifname: the interface to bring up
+        """
+        return self.command("sudo config interface startup {}".format(ifname))
+
 
 class EosHost(AnsibleHostBase):
     """

tests/common/plugins/sanity_check/__init__.py

@@ -49,15 +49,15 @@ def _update_check_items(old_items, new_items, supported_items):
 
 
 @pytest.fixture(scope="module", autouse=True)
-def sanity_check(testbed_devices, request):
+def sanity_check(testbed_devices, request, fanouthosts):
     logger.info("Start pre-test sanity check")
 
     dut = testbed_devices["dut"]
     localhost = testbed_devices["localhost"]
 
     skip_sanity = False
     allow_recover = False
-    recover_method = "config_reload"
+    recover_method = "adaptive"
     check_items = set(copy.deepcopy(constants.SUPPORTED_CHECK_ITEMS))  # Default check items
     post_check = False
 
@@ -72,7 +72,7 @@ def sanity_check(testbed_devices, request):
         logger.info("Process marker %s in script. m.args=%s, m.kwargs=%s" % (m.name, str(m.args), str(m.kwargs)))
         skip_sanity = customized_sanity_check.kwargs.get("skip_sanity", False)
         allow_recover = customized_sanity_check.kwargs.get("allow_recover", False)
-        recover_method = customized_sanity_check.kwargs.get("recover_method", "config_reload")
+        recover_method = customized_sanity_check.kwargs.get("recover_method", "adaptive")
         if allow_recover and recover_method not in constants.RECOVER_METHODS:
             pytest.warning("Unsupported recover method")
             logger.info("Fall back to use default recover method 'config_reload'")
@@ -108,11 +108,11 @@ def sanity_check(testbed_devices, request):
                 json.dumps(check_results, indent=4))
     if any([result["failed"] for result in check_results]):
         if not allow_recover:
-            pytest.fail("Pre-test sanity check failed, allow_recover=False")
+            pytest.fail("Pre-test sanity check failed, allow_recover=False {}".format(check_results))
             return
 
         logger.info("Pre-test sanity check failed, try to recover, recover_method=%s" % recover_method)
-        recover(dut, localhost, recover_method)
+        recover(dut, localhost, fanouthosts, check_results, recover_method)
         logger.info("Run sanity check again after recovery")
         new_check_results = do_checks(dut, check_items)
         logger.info("!!!!!!!!!!!!!!!! Pre-test sanity check after recovery results: !!!!!!!!!!!!!!!!\n%s" % \

tests/common/plugins/sanity_check/constants.py

@@ -12,11 +12,12 @@
 
 # Recover related definitions
 RECOVER_METHODS = {
-    "config_reload": {"cmd": "config reload -y", "reboot": False},
-    "load_minigraph": {"cmd": "config load_minigraph -y", "reboot": False},
-    "reboot": {"cmd": "reboot", "reboot": True},
-    "warm_reboot": {"cmd": "warm-reboot", "reboot": True},
-    "fast_reboot": {"cmd": "fast_reboot", "reboot": True}
+    "config_reload": {"cmd": "config reload -y", "reboot": False, "adaptive": False, 'recover_wait': 60},
+    "load_minigraph": {"cmd": "config load_minigraph -y", "reboot": False, "adaptive": False, 'recover_wait': 60},
+    "reboot": {"cmd": "reboot", "reboot": True, "adaptive": False, 'recover_wait': 120},
+    "warm_reboot": {"cmd": "warm-reboot", "reboot": True, "adaptive": False, 'recover_wait': 120},
+    "fast_reboot": {"cmd": "fast_reboot", "reboot": True, "adaptive": False, 'recover_wait': 120},
+    "adaptive": {"cmd": None, "reboot": False, "adaptive": True, 'recover_wait': 30},
 }       # All supported recover methods
 
 SUPPORTED_CHECK_ITEMS = ["services", "interfaces", "dbmemory"]          # Supported checks

tests/common/plugins/sanity_check/recover.py

@@ -1,16 +1,16 @@
 
 import logging
-import time
 
 import constants
 
-from common.utilities import wait, wait_until
+from common.utilities import wait
 from common.errors import RunAnsibleModuleFail
+from common.platform.device_utils import fanout_switch_port_lookup
 
 logger = logging.getLogger(__name__)
 
 
-def reboot_dut(dut, localhost, cmd):
+def reboot_dut(dut, localhost, cmd, wait_time):
     logger.info("Reboot dut using cmd='%s'" % cmd)
     reboot_task, reboot_res = dut.command(cmd, module_async=True)
 
@@ -26,13 +26,64 @@ def reboot_dut(dut, localhost, cmd):
         assert False, "Failed to reboot the DUT"
 
     localhost.wait_for(host=dut.hostname, port=22, state="started", delay=10, timeout=300)
-    wait(30, msg="Wait 30 seconds for system to be stable.")
+    wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time))
 
 
-def recover(dut, localhost, recover_method):
+def __recover_interfaces(dut, fanouthosts, result, wait_time):
+    for port in result['down_ports']:
+        logging.info("Restoring port {}".format(port))
+        fanout, fanout_port = fanout_switch_port_lookup(fanouthosts, port)
+        if fanout and fanout_port:
+            fanout.no_shutdown(fanout_port)
+        dut.no_shutdown(port)
+    wait(wait_time, msg="Wait {} seconds for interface(s) to restore.".format(wait_time))
+
+
+def __recover_services(dut, result):
+    status   = result['services_status']
+    services = [ x for x in status if not status[x] ]
+    logging.info("Service(s) down: {}".format(services))
+    return 'reboot' if 'database' in services else 'config_reload'
+
+
+def __recover_with_command(dut, cmd, wait_time):
+    dut.command(cmd)
+    wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time))
+
+
+def adaptive_recover(dut, localhost, fanouthosts, check_results, wait_time):
+    outstanding_action = None
+    for result in check_results:
+        if result['failed']:
+            logging.info("Restoring {}".format(result))
+            if result['check_item'] == 'interfaces':
+                __recover_interfaces(dut, fanouthosts, result, wait_time)
+            elif result['check_item'] == 'services':
+                action             = __recover_services(dut, result)
+                # Only allow outstanding_action be overridden when it is
+                # None. In case the outstanding_action has already been
+                # been set to 'reboot'.
+                if not outstanding_action:
+                    outstanding_action = action
+            else:
+                outstanding_action = 'reboot'
+
+    if outstanding_action:
+        method    = constants.RECOVER_METHODS[outstanding_action]
+        wait_time = method['recover_wait']
+        if method["reboot"]:
+            reboot_dut(dut, localhost, method["cmd"], wait_time)
+        else:
+            __recover_with_command(dut, method['cmd'], wait_time)
+
+
+def recover(dut, localhost, fanouthosts, check_results, recover_method):
     logger.info("Try to recover %s using method %s" % (dut.hostname, recover_method))
-    if constants.RECOVER_METHODS[recover_method]["reboot"]:
-        reboot_dut(dut, localhost, constants.RECOVER_METHODS[recover_method]["cmd"])
+    method    = constants.RECOVER_METHODS[recover_method]
+    wait_time = method['recover_wait']
+    if method["adaptive"]:
+        adaptive_recover(dut, localhost, fanouthosts, check_results, wait_time)
+    elif method["reboot"]:
+        reboot_dut(dut, localhost, method["cmd"], wait_time)
     else:
-        dut.command(constants.RECOVER_METHODS[recover_method]["cmd"])
-        wait(30, msg="Wait 30 seconds for system to be stable.")
+        __recover_with_command(dut, method['cmd'], wait_time)