checking disk and memory space before installing image (#11151)

What is the motivation for this PR?
Update images failed sometimes on low-memory devices.

How did you do it?
Check memory and disk space before installing image.
If no enough space of memory or disk, abort.

How did you verify/test it?
Upgrade the images on low memory and small disk devices several times. no issue.

Signed-off-by: xuliping <[email protected]>

Author: lipxu

ansible/library/reduce_and_add_sonic_images.py

@@ -143,7 +143,9 @@ def download_new_sonic_image(module, new_image_url, save_as):
         log("Completed downloading image")
 
     free_disk_size = get_disk_free_size(module, "/")
-    log("After downloaded sonic image, latest free disk size: {}".format(free_disk_size))
+    total, avail = get_memory_sizes(module)
+    log("After downloaded sonic image, latest free disk size: {}, "
+        "memory total {} available {}".format(free_disk_size, total, avail))
 
     if path.exists(save_as):
         log("Checking downloaded image version")
@@ -152,13 +154,35 @@ def download_new_sonic_image(module, new_image_url, save_as):
         log("Downloaded image version: {}".format(results["downloaded_image_version"]))
 
 
-def install_new_sonic_image(module, new_image_url, save_as=None):
+def install_new_sonic_image(module, new_image_url, save_as=None, required_space=1600):
     log("install new sonic image")
 
+    log("Clean-up previous downloads first")
+    exec_command(
+        module,
+        cmd="rm -f {}".format("/host/downloaded-sonic-image"),
+        msg="clean up previously downloaded image",
+        ignore_error=True
+    )
+
     if not save_as:
         avail = get_disk_free_size(module, "/host")
         save_as = "/host/downloaded-sonic-image" if avail >= 2000 else "/tmp/tmpfs/downloaded-sonic-image"
 
+    free_disk_size = get_disk_free_size(module, "/")
+    total, avail = get_memory_sizes(module)
+    log("Before install sonic image, free disk {}, memory total {} available {}".format(free_disk_size, total, avail))
+    if avail < 1024 or free_disk_size < required_space:
+        log("free memory or disk space size is not enough to install a new image")
+        module.fail_json(
+            msg="Image installation failed: rc=%d, out=%s, err=%s" % (
+                -1,
+                "free memory or disk space size is not enough to install a new image",
+                ""
+            )
+        )
+        return
+
     if save_as.startswith("/tmp/tmpfs"):
         log("Create a tmpfs partition to download image to install")
         exec_command(module, cmd="mkdir -p /tmp/tmpfs", ignore_error=True)
@@ -228,9 +252,14 @@ def get_disk_used_percent(module):
         output = exec_command(module, cmd="df -BM --output=pcent /host")[1]
         return int(output.splitlines()[-1][:-1])
 
+    def get_disk_total_size(module):
+        output = exec_command(module, cmd="df -BM --output=size /host")[1]
+        return int(output.splitlines()[-1][:-1])
+
     current_used_percent = get_disk_used_percent(module)
-    log("current used percent: {}".format(current_used_percent))
-    if current_used_percent > disk_used_pcent:
+    total_size = get_disk_total_size(module)
+    log("current used percent: {}, total_size {}".format(current_used_percent, total_size))
+    if current_used_percent > disk_used_pcent or total_size < 4096:
         log("Trying to free up spaces at best effort")
         exec_command(module, "rm -f /var/log/*.gz", ignore_error=True)
         exec_command(module, "rm -f /var/core/*", ignore_error=True)
@@ -245,6 +274,58 @@ def get_disk_used_percent(module):
     log("After free up disk space, latest free disk size: {}".format(free_disk_size))
 
 
+def free_up_memory_drop_caches(module):
+    """
+    To free pagecache:
+        echo 1 > /proc/sys/vm/drop_caches
+    To free reclaimable slab objects (includes dentries and inodes):
+        echo 2 > /proc/sys/vm/drop_caches
+    To free slab objects and pagecache:
+        echo 3 > /proc/sys/vm/drop_caches
+    """
+    for index in range(1, 4):
+        cmd = 'echo {} >  /proc/sys/vm/drop_caches'.format(index)
+        exec_command(module, cmd, ignore_error=True)
+        cmd = 'sync'
+        exec_command(module, cmd, ignore_error=True)
+        log("drop {} cache to free up memory space".format(index))
+    return
+
+
+def free_up_memory_stop_process(module):
+    """
+    stop the process which not need during upgrade
+    not start it here since it will be started after reboot
+    sudo systemctl list-units --type=service --state=running
+    """
+
+    processes = ['monit.service', 'telemetry.service', 'stop pmon.service']
+    for process in processes:
+        cmd = 'systemctl stop {}'.format(process)
+        exec_command(module, cmd, ignore_error=True)
+        log("stop process {}".format(process))
+
+    return
+
+
+def free_up_memory_space(module, free_space_needed=1024):
+    """for low memory device, drop cache and disable process which not need during upgrade."""
+    log("free up memory space at best effort")
+
+    total, avail = get_memory_sizes(module)
+    if avail > free_space_needed:
+        log("Available memory {}, no need to free up memory space".format(avail))
+        return
+
+    free_up_memory_drop_caches(module)
+    free_up_memory_stop_process(module)
+
+    total, avail = get_memory_sizes(module)
+    log("After free up, current available memory {}".format(avail))
+
+    return
+
+
 def work_around_for_reboot(module):
     # work around reboot for s6100
     # Replace /usr/share/sonic/device/x86_64-dell_s6100_c2538-r0/platform_reboot_pre_check
@@ -275,12 +356,14 @@ def main():
             disk_used_pcent=dict(required=False, type='int', default=8),
             new_image_url=dict(required=False, type='str', default=None),
             save_as=dict(required=False, type='str', default=None),
+            required_space=dict(required=False, type='int', default=1600),
         ),
         supports_check_mode=False)
 
     disk_used_pcent = module.params['disk_used_pcent']
     new_image_url = module.params['new_image_url']
     save_as = module.params['save_as']
+    required_space = module.params['required_space']
 
     try:
         if not new_image_url:
@@ -295,10 +378,11 @@ def main():
             results["current_stage"] = "prepare"
 
             free_up_disk_space(module, disk_used_pcent)
+            free_up_memory_space(module)
             setup_swap_if_necessary(module)
             results["current_stage"] = "install"
 
-            install_new_sonic_image(module, new_image_url, save_as)
+            install_new_sonic_image(module, new_image_url, save_as, required_space)
             results["current_stage"] = "complete"
     except Exception:
         err = str(sys.exc_info())