From 5e73ddb8e684f5daf02b1a3a74c54cd762a64c06 Mon Sep 17 00:00:00 2001 From: Jipan Yang Date: Fri, 3 Aug 2018 16:18:33 -0700 Subject: [PATCH 1/5] Add support for docker upgrade Signed-off-by: Jipan Yang --- sonic_installer/main.py | 63 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 5b2933a0b8..84d950356d 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -317,5 +317,68 @@ def cleanup(): if image_removed == 0: click.echo("No image(s) to remove") +# Upgrade docker image +@cli.command() +@click.option('-y', '--yes', is_flag=True, callback=abort_if_false, + expose_value=False, prompt='New docker image will be installed, continue?') +@click.option('--cleanup_image', is_flag=True, help="Clean up old docker images") +@click.argument('container_name', metavar='', required=True, type=click.Choice(["swss", "snmp", "lldp"])) +@click.argument('tag', metavar='', required=True, type=click.STRING) +@click.argument('url') +def upgrade_docker(cleanup_image, container_name, tag, url): + """ Upgrade docker image from local binary or URL""" + + # example images: docker-lldp-sv2:latest + cmd = "docker inspect --format '{{.Config.Image}}' " + container_name + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + images = proc.stdout.read().rstrip() + + # example image_name: docker-lldp-sv2 + cmd = "echo " + images + " | cut -d ':' -f 1" + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + image_name = proc.stdout.read().rstrip() + + # example image_id: 7ed919240fb9 + cmd = "docker images --format '{{.ID}}' " + image_name + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + image_id = proc.stdout.read() + + + DEFAULT_IMAGE_PATH = os.path.join("/tmp/", image_name) + + if url.startswith('http://') or url.startswith('https://'): + click.echo('Downloading image...') + urllib.urlretrieve(url, DEFAULT_IMAGE_PATH, reporthook) + image_path = DEFAULT_IMAGE_PATH + else: + image_path = os.path.join("./", url) + + # make sure orchagent is in clean state if swss is to be upgraded + if container_name == "swss": + cmd = "docker exec -it " + container_name + " orchagent_restart_check" + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + result = proc.stdout.read().rstrip() + if result != "RESTARTCHECK succeeded": + click.echo("Orchagent is not in clean state, check syslog and try later") + sys.exit(1) + else: + click.echo("Orchagent is in clean state and frozen for warm upgrade") + + run_command("systemctl stop %s" % container_name) + run_command("docker rm %s" % container_name) + run_command("docker rmi %s" % images) + run_command("docker load < %s" % image_path) + run_command("docker tag %s:latest %s:%s" % (image_name, image_name, tag)) + run_command("systemctl restart %s" % container_name) + # Clean up old docker images + if cleanup_image: + image_id = image_id.splitlines() + image_id = set(image_id) + for id in image_id: + run_command("docker rmi -f %s" % id) + + run_command("sleep 5") # wait 5 seconds for application to sync + click.echo('Done') + if __name__ == '__main__': cli() From 502b9a78f4567fb36039a7dc9d93c7dafe6e7739 Mon Sep 17 00:00:00 2001 From: Jipan Yang Date: Sun, 5 Aug 2018 21:27:06 -0700 Subject: [PATCH 2/5] Remove the extra spaces Signed-off-by: Jipan Yang --- sonic_installer/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 84d950356d..36759c7049 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -339,7 +339,7 @@ def upgrade_docker(cleanup_image, container_name, tag, url): image_name = proc.stdout.read().rstrip() # example image_id: 7ed919240fb9 - cmd = "docker images --format '{{.ID}}' " + image_name + cmd = "docker images --format '{{.ID}}' " + image_name proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) image_id = proc.stdout.read() @@ -355,7 +355,7 @@ def upgrade_docker(cleanup_image, container_name, tag, url): # make sure orchagent is in clean state if swss is to be upgraded if container_name == "swss": - cmd = "docker exec -it " + container_name + " orchagent_restart_check" + cmd = "docker exec -it " + container_name + " orchagent_restart_check" proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) result = proc.stdout.read().rstrip() if result != "RESTARTCHECK succeeded": From e3a7c445f41ab9f36e5917f9773a18bdc9da2077 Mon Sep 17 00:00:00 2001 From: Jipan Yang Date: Mon, 17 Sep 2018 19:11:08 -0700 Subject: [PATCH 3/5] Make pending task check optional and add function support to get docker tag from docker meta label Signed-off-by: Jipan Yang --- sonic_installer/main.py | 86 +++++++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 36759c7049..83d1835a0f 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -147,6 +147,20 @@ def remove_image(image): run_command('grub-set-default --boot-directory=' + HOST_PATH + ' 0') click.echo('Image removed') +# TODO: Embed tag name info into docker image meta data at build time, +# and extract tag name from docker image file. +def get_docker_tag_name(image): + # Try to get tag name from label metadata + cmd = "docker inspect --format '{{.ContainerConfig.Labels.Tag}}' " + image + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + (out, err) = proc.communicate() + if proc.returncode != 0: + return "unknown" + tag = out.rstrip() + if tag == "": + return "unknown" + return tag + # Callback for confirmation prompt. Aborts if user enters "n" def abort_if_false(ctx, param, value): if not value: @@ -321,61 +335,85 @@ def cleanup(): @cli.command() @click.option('-y', '--yes', is_flag=True, callback=abort_if_false, expose_value=False, prompt='New docker image will be installed, continue?') -@click.option('--cleanup_image', is_flag=True, help="Clean up old docker images") -@click.argument('container_name', metavar='', required=True, type=click.Choice(["swss", "snmp", "lldp"])) -@click.argument('tag', metavar='', required=True, type=click.STRING) +@click.option('--cleanup_image', is_flag=True, help="Clean up old docker image") +@click.option('--enforce_check', is_flag=True, help="Enforce pending task check for docker upgrade") +@click.option('--tag', type=str, help="Tag for the new docker image") +@click.argument('container_name', metavar='', required=True, + type=click.Choice(["swss", "snmp", "lldp", "bgp", "pmon", "dhcp_relay", "telemetry", "teamd"])) @click.argument('url') -def upgrade_docker(cleanup_image, container_name, tag, url): +def upgrade_docker(container_name, url, cleanup_image, enforce_check, tag): """ Upgrade docker image from local binary or URL""" - # example images: docker-lldp-sv2:latest + # example image: docker-lldp-sv2:latest cmd = "docker inspect --format '{{.Config.Image}}' " + container_name proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) - images = proc.stdout.read().rstrip() + (out, err) = proc.communicate() + if proc.returncode != 0: + sys.exit(proc.returncode) + image_latest = out.rstrip() # example image_name: docker-lldp-sv2 - cmd = "echo " + images + " | cut -d ':' -f 1" + cmd = "echo " + image_latest + " | cut -d ':' -f 1" proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) image_name = proc.stdout.read().rstrip() - # example image_id: 7ed919240fb9 - cmd = "docker images --format '{{.ID}}' " + image_name - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) - image_id = proc.stdout.read() - - DEFAULT_IMAGE_PATH = os.path.join("/tmp/", image_name) - if url.startswith('http://') or url.startswith('https://'): click.echo('Downloading image...') - urllib.urlretrieve(url, DEFAULT_IMAGE_PATH, reporthook) + try: + urllib.urlretrieve(url, DEFAULT_IMAGE_PATH, reporthook) + except Exception, e: + click.echo("Download error", e) + return image_path = DEFAULT_IMAGE_PATH else: image_path = os.path.join("./", url) # make sure orchagent is in clean state if swss is to be upgraded if container_name == "swss": - cmd = "docker exec -it " + container_name + " orchagent_restart_check" + skipPendingTaskCheck = " -s" + if enforce_check: + skipPendingTaskCheck = "" + + cmd = "docker exec -it " + container_name + " orchagent_restart_check" + skipPendingTaskCheck proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) result = proc.stdout.read().rstrip() if result != "RESTARTCHECK succeeded": - click.echo("Orchagent is not in clean state, check syslog and try later") - sys.exit(1) + if enforce_check: + click.echo("Orchagent is not in clean state, check syslog and try later") + sys.exit(1) + else: + click.echo("Orchagent is not in clean state, upgrading it anyway") else: click.echo("Orchagent is in clean state and frozen for warm upgrade") run_command("systemctl stop %s" % container_name) - run_command("docker rm %s" % container_name) - run_command("docker rmi %s" % images) + run_command("docker rm %s " % container_name) + run_command("docker rmi %s " % image_latest) run_command("docker load < %s" % image_path) + if tag == None: + # example image: docker-lldp-sv2:latest + tag = get_docker_tag_name(image_latest) run_command("docker tag %s:latest %s:%s" % (image_name, image_name, tag)) run_command("systemctl restart %s" % container_name) + # Clean up old docker images if cleanup_image: - image_id = image_id.splitlines() - image_id = set(image_id) - for id in image_id: - run_command("docker rmi -f %s" % id) + # All images id under the image name + cmd = "docker images --format '{{.ID}}' " + image_name + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + image_id_all = proc.stdout.read() + image_id_all = image_id_all.splitlines() + image_id_all = set(image_id_all) + + # this is image_id for image with "latest" tag + cmd = "docker images --format '{{.ID}}' " + image_latest + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + image_id_latest = proc.stdout.read().rstrip() + + for id in image_id_all: + if id != image_id_latest: + run_command("docker rmi -f %s" % id) run_command("sleep 5") # wait 5 seconds for application to sync click.echo('Done') From 6130acc6b275a74087b6cbb4c1df545e9d249645 Mon Sep 17 00:00:00 2001 From: Jipan Yang Date: Tue, 18 Sep 2018 12:41:43 -0700 Subject: [PATCH 4/5] Use proc.returncode instead proc result for orchagent_restart_check Signed-off-by: Jipan Yang --- sonic_installer/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 83d1835a0f..e07f9487fc 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -377,11 +377,11 @@ def upgrade_docker(container_name, url, cleanup_image, enforce_check, tag): cmd = "docker exec -it " + container_name + " orchagent_restart_check" + skipPendingTaskCheck proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) - result = proc.stdout.read().rstrip() - if result != "RESTARTCHECK succeeded": + (out, err) = proc.communicate() + if proc.returncode != 0: if enforce_check: click.echo("Orchagent is not in clean state, check syslog and try later") - sys.exit(1) + sys.exit(proc.returncode) else: click.echo("Orchagent is not in clean state, upgrading it anyway") else: From 48474677c7b61500e93cab0ec173c1fa2d6b1f9b Mon Sep 17 00:00:00 2001 From: Jipan Yang Date: Wed, 26 Sep 2018 11:54:19 -0700 Subject: [PATCH 5/5] Remove extra spaces Signed-off-by: Jipan Yang --- sonic_installer/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index e07f9487fc..71cf3513b8 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -151,7 +151,7 @@ def remove_image(image): # and extract tag name from docker image file. def get_docker_tag_name(image): # Try to get tag name from label metadata - cmd = "docker inspect --format '{{.ContainerConfig.Labels.Tag}}' " + image + cmd = "docker inspect --format '{{.ContainerConfig.Labels.Tag}}' " + image proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) (out, err) = proc.communicate() if proc.returncode != 0: @@ -388,8 +388,8 @@ def upgrade_docker(container_name, url, cleanup_image, enforce_check, tag): click.echo("Orchagent is in clean state and frozen for warm upgrade") run_command("systemctl stop %s" % container_name) - run_command("docker rm %s " % container_name) - run_command("docker rmi %s " % image_latest) + run_command("docker rm %s " % container_name) + run_command("docker rmi %s " % image_latest) run_command("docker load < %s" % image_path) if tag == None: # example image: docker-lldp-sv2:latest