From 23568691803da584507d05b83cb95eb90cabcba0 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 3 Nov 2020 12:41:24 +0200 Subject: [PATCH 01/60] [sonic-installer] migrate SONiC packages Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 95 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 85 insertions(+), 10 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index ca843c394b..43da10f11f 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -213,17 +213,48 @@ def print_deprecation_warning(deprecated_cmd_or_subcmd, new_cmd_or_subcmd): fg="red", err=True) click.secho("Please use '{}' instead".format(new_cmd_or_subcmd), fg="red", err=True) + +def mount_squash_fs(squashfs_path, mount_point): + run_command_or_raise(["mkdir", "-p", mount_point]) + run_command_or_raise(["mount", "-t", "squashfs", squashfs_path, mount_point]) + + +def umount(mount_point, read_only=True, recursive=False, force=True, remove_dir=True): + flags = "-" + if read_only: + flags = flags + "r" + if force: + flags = flags + "f" + if recursive: + flags = flags + "R" + run_command_or_raise(["umount", flags, mount_point]) + if remove_dir: + run_command_or_raise(["rm", "-rf", mount_point]) + + +def mount_overlay_fs(lowerdir, upperdir, workdir, mount_point): + run_command_or_raise(["mkdir", "-p", mount_point]) + overlay_options = "rw,relatime,lowerdir={},upperdir={},workdir={}".format(lowerdir, upperdir, workdir) + run_command_or_raise(["mount", "overlay", "-t", "overlay", "-o", overlay_options, mount_point]) + + +def mount_bind(source, mount_point): + run_command_or_raise(["mkdir", "-p", mount_point]) + run_command_or_raise(["mount", "--bind", source, mount_point]) + + +def mount_procfs_chroot(root): + run_command_or_raise(["chroot", root, "mount", "proc", "/proc", "-t", "proc"]) + + +def mount_sysfs_chroot(root): + run_command_or_raise(["chroot", root, "mount", "sysfs", "/sys", "-t", "sysfs"]) + + def update_sonic_environment(click, binary_image_version): """Prepare sonic environment variable using incoming image template file. If incoming image template does not exist use current image template file. """ - def mount_next_image_fs(squashfs_path, mount_point): - run_command_or_raise(["mkdir", "-p", mount_point]) - run_command_or_raise(["mount", "-t", "squashfs", squashfs_path, mount_point]) - - def umount_next_image_fs(mount_point): - run_command_or_raise(["umount", "-rf", mount_point]) - run_command_or_raise(["rm", "-rf", mount_point]) SONIC_ENV_TEMPLATE_FILE = os.path.join("usr", "share", "sonic", "templates", "sonic-environment.j2") SONIC_VERSION_YML_FILE = os.path.join("etc", "sonic", "sonic_version.yml") @@ -236,7 +267,7 @@ def umount_next_image_fs(mount_point): env_file = os.path.join(env_dir, "sonic-environment") try: - mount_next_image_fs(new_image_squashfs_path, new_image_mount) + mount_squash_fs(new_image_squashfs_path, new_image_mount) next_sonic_env_template_file = os.path.join(new_image_mount, SONIC_ENV_TEMPLATE_FILE) next_sonic_version_yml_file = os.path.join(new_image_mount, SONIC_VERSION_YML_FILE) @@ -260,7 +291,46 @@ def umount_next_image_fs(mount_point): os.remove(env_file) os.rmdir(env_dir) finally: - umount_next_image_fs(new_image_mount) + umount(new_image_mount) + + +def migrate_sonic_packages(image_version): + """ Migrate SONiC packages to new SONiC image. """ + + SONIC_PACKAGE_MANAGER = "sonic-package-manager" + PACKAGE_MANAGER_DIR = "/var/lib/sonic-package-manager/" + DOCKER_CTL_SCRIPT = "/usr/lib/docker/docker.sh" + + sonic_version = re.sub("SONiC-OS-", '', image_version) + new_image_dir = os.path.join('/', "host", "image-{0}".format(sonic_version)) + new_image_squashfs_path = os.path.join(new_image_dir, "fs.squashfs") + new_image_upper_dir = os.path.join(new_image_dir, "rw") + new_image_work_dir = os.path.join(new_image_dir, "work") + new_image_docker_dir = os.path.join(new_image_dir, "docker") + new_image_mount = os.path.join('/', "tmp", "image-{0}-fs".format(sonic_version)) + new_image_docker_mount = os.path.join(new_image_mount, "var", "lib", "docker") + + try: + packages_file = "packages.json" + packages_path = os.path.join(PACKAGE_MANAGER_DIR, packages_file) + tmp_dir = "/tmp" + mount_squash_fs(new_image_squashfs_path, new_image_mount) + # make sure upper dir and work dir exist + run_command_or_raise(["mkdir", "-p", new_image_upper_dir]) + run_command_or_raise(["mkdir", "-p", new_image_work_dir]) + mount_overlay_fs(new_image_mount, new_image_upper_dir, new_image_work_dir, new_image_mount) + mount_bind(new_image_docker_dir, new_image_docker_mount) + mount_procfs_chroot(new_image_mount) + mount_sysfs_chroot(new_image_mount) + run_command_or_raise(["chroot", new_image_mount, DOCKER_CTL_SCRIPT, "start"]) + run_command_or_raise(["cp", packages_path, os.path.join(new_image_mount, tmp_dir, packages_file)]) + run_command_or_raise(["chroot", new_image_mount, SONIC_PACKAGE_MANAGER, "migrate", + "{}".format(os.path.join(tmp_dir, packages_file)), "-y"]) + finally: + run_command("chroot {} {} stop".format(new_image_mount, DOCKER_CTL_SCRIPT)) + umount(new_image_mount, recursive=True, read_only=False, remove_dir=False) + umount(new_image_mount) + # Main entrypoint @click.group(cls=AliasedGroup) @@ -282,8 +352,10 @@ def sonic_installer(): help="Force installation of an image of a type which differs from that of the current running image") @click.option('--skip_migration', is_flag=True, help="Do not migrate current configuration to the newly installed image") +@click.option('--skip-package-migration', is_flag=True, + help="Do not migrate current packages to the newly installed image") @click.argument('url') -def install(url, force, skip_migration=False): +def install(url, force, skip_migration=False, skip_package_migration=False): """ Install image from local binary or URL""" bootloader = get_bootloader() @@ -329,6 +401,9 @@ def install(url, force, skip_migration=False): update_sonic_environment(click, binary_image_version) + if not skip_package_migration: + migrate_sonic_packages(binary_image_version) + # Finally, sync filesystem run_command("sync;sync;sync") run_command("sleep 3") # wait 3 seconds after sync From ece38450f94a49bca688e06f1d92f8592e616ccf Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 3 Nov 2020 16:36:41 +0200 Subject: [PATCH 02/60] fix temp dir path for chroot Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 43da10f11f..3930e6d59a 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -313,7 +313,7 @@ def migrate_sonic_packages(image_version): try: packages_file = "packages.json" packages_path = os.path.join(PACKAGE_MANAGER_DIR, packages_file) - tmp_dir = "/tmp" + tmp_dir = "tmp" mount_squash_fs(new_image_squashfs_path, new_image_mount) # make sure upper dir and work dir exist run_command_or_raise(["mkdir", "-p", new_image_upper_dir]) @@ -325,7 +325,7 @@ def migrate_sonic_packages(image_version): run_command_or_raise(["chroot", new_image_mount, DOCKER_CTL_SCRIPT, "start"]) run_command_or_raise(["cp", packages_path, os.path.join(new_image_mount, tmp_dir, packages_file)]) run_command_or_raise(["chroot", new_image_mount, SONIC_PACKAGE_MANAGER, "migrate", - "{}".format(os.path.join(tmp_dir, packages_file)), "-y"]) + "{}".format(os.path.join("/", tmp_dir, packages_file)), "-y"]) finally: run_command("chroot {} {} stop".format(new_image_mount, DOCKER_CTL_SCRIPT)) umount(new_image_mount, recursive=True, read_only=False, remove_dir=False) From 8bc75a769e3a4cd3b1a12876e189e412c9b70cca Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 3 Nov 2020 19:45:01 +0200 Subject: [PATCH 03/60] fix for install of already installed image Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 3930e6d59a..919854fdaa 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -401,8 +401,8 @@ def install(url, force, skip_migration=False, skip_package_migration=False): update_sonic_environment(click, binary_image_version) - if not skip_package_migration: - migrate_sonic_packages(binary_image_version) + if not skip_package_migration: + migrate_sonic_packages(binary_image_version) # Finally, sync filesystem run_command("sync;sync;sync") From e72f54059bb8538b993a8a06f0e03031582b255f Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Tue, 15 Dec 2020 19:14:39 +0200 Subject: [PATCH 04/60] move constants to the top Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 919854fdaa..661419ead8 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -301,19 +301,19 @@ def migrate_sonic_packages(image_version): PACKAGE_MANAGER_DIR = "/var/lib/sonic-package-manager/" DOCKER_CTL_SCRIPT = "/usr/lib/docker/docker.sh" + tmp_dir = "tmp" + packages_file = "packages.json" + packages_path = os.path.join(PACKAGE_MANAGER_DIR, packages_file) sonic_version = re.sub("SONiC-OS-", '', image_version) new_image_dir = os.path.join('/', "host", "image-{0}".format(sonic_version)) new_image_squashfs_path = os.path.join(new_image_dir, "fs.squashfs") new_image_upper_dir = os.path.join(new_image_dir, "rw") new_image_work_dir = os.path.join(new_image_dir, "work") new_image_docker_dir = os.path.join(new_image_dir, "docker") - new_image_mount = os.path.join('/', "tmp", "image-{0}-fs".format(sonic_version)) + new_image_mount = os.path.join('/', tmp_dir, "image-{0}-fs".format(sonic_version)) new_image_docker_mount = os.path.join(new_image_mount, "var", "lib", "docker") try: - packages_file = "packages.json" - packages_path = os.path.join(PACKAGE_MANAGER_DIR, packages_file) - tmp_dir = "tmp" mount_squash_fs(new_image_squashfs_path, new_image_mount) # make sure upper dir and work dir exist run_command_or_raise(["mkdir", "-p", new_image_upper_dir]) From be4f089e635494328a21f01c671da50efd6bc80d Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak Date: Thu, 21 Jan 2021 15:36:22 +0200 Subject: [PATCH 05/60] [sonic_installer] migrate packages from old docker library Signed-off-by: Stepan Blyshchak --- sonic_installer/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sonic_installer/main.py b/sonic_installer/main.py index 9de42f3a3d..e26d188600 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -294,6 +294,8 @@ def migrate_sonic_packages(image_version): SONIC_PACKAGE_MANAGER = "sonic-package-manager" PACKAGE_MANAGER_DIR = "/var/lib/sonic-package-manager/" DOCKER_CTL_SCRIPT = "/usr/lib/docker/docker.sh" + DOCKERD_SOCK = "docker.sock" + VAR_RUN_PATH = "/var/run/" tmp_dir = "tmp" packages_file = "packages.json" @@ -318,8 +320,14 @@ def migrate_sonic_packages(image_version): mount_sysfs_chroot(new_image_mount) run_command_or_raise(["chroot", new_image_mount, DOCKER_CTL_SCRIPT, "start"]) run_command_or_raise(["cp", packages_path, os.path.join(new_image_mount, tmp_dir, packages_file)]) + run_command_or_raise(["touch", os.path.join(new_image_mount, "tmp", DOCKERD_SOCK)]) + run_command_or_raise(["mount", "--bind", + os.path.join(VAR_RUN_PATH, DOCKERD_SOCK), + os.path.join(new_image_mount, "tmp", DOCKERD_SOCK)]) run_command_or_raise(["chroot", new_image_mount, SONIC_PACKAGE_MANAGER, "migrate", - "{}".format(os.path.join("/", tmp_dir, packages_file)), "-y"]) + os.path.join("/", tmp_dir, packages_file), + "--dockerd-socket", os.path.join("/", tmp_dir, DOCKERD_SOCK), + "-y"]) finally: run_command("chroot {} {} stop".format(new_image_mount, DOCKER_CTL_SCRIPT)) umount(new_image_mount, recursive=True, read_only=False, remove_dir=False) From 9fd283d3656cf0741b43d70be6b88992bf3cf51d Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 18 Mar 2021 15:58:55 +0200 Subject: [PATCH 06/60] remove unused imports Signed-off-by: Stepan Blyschak --- sonic_installer/bootloader/aboot.py | 1 - sonic_installer/bootloader/bootloader.py | 1 - 2 files changed, 2 deletions(-) diff --git a/sonic_installer/bootloader/aboot.py b/sonic_installer/bootloader/aboot.py index 3dbb29c2f7..ef1ea622cd 100644 --- a/sonic_installer/bootloader/aboot.py +++ b/sonic_installer/bootloader/aboot.py @@ -19,7 +19,6 @@ HOST_PATH, IMAGE_DIR_PREFIX, IMAGE_PREFIX, - ROOTFS_NAME, run_command, run_command_or_raise, ) diff --git a/sonic_installer/bootloader/bootloader.py b/sonic_installer/bootloader/bootloader.py index fba5c79f49..8ce2041f0a 100644 --- a/sonic_installer/bootloader/bootloader.py +++ b/sonic_installer/bootloader/bootloader.py @@ -9,7 +9,6 @@ HOST_PATH, IMAGE_DIR_PREFIX, IMAGE_PREFIX, - ROOTFS_NAME, ) class Bootloader(object): From 0cbaa3e2feced78f843f91c6f999ca7d96a22360 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 19 Mar 2021 11:27:45 +0200 Subject: [PATCH 07/60] fix redefining path Signed-off-by: Stepan Blyschak --- sonic_installer/bootloader/bootloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_installer/bootloader/bootloader.py b/sonic_installer/bootloader/bootloader.py index 8ce2041f0a..607cd730db 100644 --- a/sonic_installer/bootloader/bootloader.py +++ b/sonic_installer/bootloader/bootloader.py @@ -70,6 +70,6 @@ def get_image_path(cls, image): return image.replace(IMAGE_PREFIX, prefix) @contextmanager - def get_path_in_image(self, image_path, path): + def get_path_in_image(self, image_path, path_in_image): """returns the path to the squashfs""" - yield path.join(image_path, path) \ No newline at end of file + yield path.join(image_path, path_in_image) \ No newline at end of file From 1a9979737f1a4980a60cac3c1620e155812d2185 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 23 Mar 2021 10:21:26 +0000 Subject: [PATCH 08/60] [sonic-package-manager] add new utility to manage SONiC packages Signed-off-by: Stepan Blyschak --- setup.py | 15 + .../bash_completion.d/sonic-package-manager | 8 + sonic-utilities-data/bash_completion.d/spm | 8 + sonic-utilities-data/templates/dump.sh.j2 | 29 + sonic-utilities-data/templates/monit.conf.j2 | 18 + .../templates/service_mgmt.sh.j2 | 149 +++ .../templates/sonic.service.j2 | 39 + sonic-utilities-data/templates/timer.unit.j2 | 15 + sonic_package_manager/__init__.py | 5 + sonic_package_manager/constraint.py | 140 +++ sonic_package_manager/database.py | 222 +++++ sonic_package_manager/dockerapi.py | 226 +++++ sonic_package_manager/errors.py | 146 +++ sonic_package_manager/logger.py | 29 + sonic_package_manager/main.py | 453 +++++++++ sonic_package_manager/manager.py | 896 ++++++++++++++++++ sonic_package_manager/manifest.py | 210 ++++ sonic_package_manager/metadata.py | 185 ++++ sonic_package_manager/package.py | 53 ++ sonic_package_manager/progress.py | 52 + sonic_package_manager/reference.py | 30 + sonic_package_manager/registry.py | 156 +++ .../service_creator/__init__.py | 3 + .../service_creator/creator.py | 339 +++++++ .../service_creator/feature.py | 108 +++ .../service_creator/sonic_db.py | 97 ++ .../service_creator/utils.py | 17 + sonic_package_manager/source.py | 178 ++++ sonic_package_manager/utils.py | 42 + sonic_package_manager/version.py | 23 + tests/sonic_package_manager/conftest.py | 379 ++++++++ tests/sonic_package_manager/test_cli.py | 63 ++ .../sonic_package_manager/test_constraint.py | 75 ++ tests/sonic_package_manager/test_database.py | 89 ++ tests/sonic_package_manager/test_manager.py | 321 +++++++ tests/sonic_package_manager/test_manifest.py | 67 ++ tests/sonic_package_manager/test_metadata.py | 36 + tests/sonic_package_manager/test_reference.py | 17 + tests/sonic_package_manager/test_registry.py | 15 + .../test_service_creator.py | 173 ++++ tests/sonic_package_manager/test_utils.py | 8 + 41 files changed, 5134 insertions(+) create mode 100644 sonic-utilities-data/bash_completion.d/sonic-package-manager create mode 100644 sonic-utilities-data/bash_completion.d/spm create mode 100644 sonic-utilities-data/templates/dump.sh.j2 create mode 100644 sonic-utilities-data/templates/monit.conf.j2 create mode 100644 sonic-utilities-data/templates/service_mgmt.sh.j2 create mode 100644 sonic-utilities-data/templates/sonic.service.j2 create mode 100644 sonic-utilities-data/templates/timer.unit.j2 create mode 100644 sonic_package_manager/__init__.py create mode 100644 sonic_package_manager/constraint.py create mode 100644 sonic_package_manager/database.py create mode 100644 sonic_package_manager/dockerapi.py create mode 100644 sonic_package_manager/errors.py create mode 100644 sonic_package_manager/logger.py create mode 100644 sonic_package_manager/main.py create mode 100644 sonic_package_manager/manager.py create mode 100644 sonic_package_manager/manifest.py create mode 100644 sonic_package_manager/metadata.py create mode 100644 sonic_package_manager/package.py create mode 100644 sonic_package_manager/progress.py create mode 100644 sonic_package_manager/reference.py create mode 100644 sonic_package_manager/registry.py create mode 100644 sonic_package_manager/service_creator/__init__.py create mode 100644 sonic_package_manager/service_creator/creator.py create mode 100644 sonic_package_manager/service_creator/feature.py create mode 100644 sonic_package_manager/service_creator/sonic_db.py create mode 100644 sonic_package_manager/service_creator/utils.py create mode 100644 sonic_package_manager/source.py create mode 100644 sonic_package_manager/utils.py create mode 100644 sonic_package_manager/version.py create mode 100644 tests/sonic_package_manager/conftest.py create mode 100644 tests/sonic_package_manager/test_cli.py create mode 100644 tests/sonic_package_manager/test_constraint.py create mode 100644 tests/sonic_package_manager/test_database.py create mode 100644 tests/sonic_package_manager/test_manager.py create mode 100644 tests/sonic_package_manager/test_manifest.py create mode 100644 tests/sonic_package_manager/test_metadata.py create mode 100644 tests/sonic_package_manager/test_reference.py create mode 100644 tests/sonic_package_manager/test_registry.py create mode 100644 tests/sonic_package_manager/test_service_creator.py create mode 100644 tests/sonic_package_manager/test_utils.py diff --git a/setup.py b/setup.py index 8018efd82c..9847435671 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,8 @@ 'show.interfaces', 'sonic_installer', 'sonic_installer.bootloader', + 'sonic_package_manager', + 'sonic_package_manager.service_creator', 'tests', 'undebug', 'utilities_common', @@ -146,19 +148,29 @@ 'sonic-clear = clear.main:cli', 'sonic-installer = sonic_installer.main:sonic_installer', 'sonic_installer = sonic_installer.main:sonic_installer', # Deprecated + 'sonic-package-manager = sonic_package_manager.main:cli', + 'spm = sonic_package_manager.main:cli', 'undebug = undebug.main:cli', 'watchdogutil = watchdogutil.main:watchdogutil', ] }, install_requires=[ 'click==7.0', + 'click-log==0.3.2', + 'docker==4.4.4', + 'docker-image-py==0.1.10', + 'filelock==3.0.12', + 'enlighten==1.8.0', 'ipaddress==1.0.23', + 'jinja2==2.11.3', 'jsondiff==1.2.0', 'm2crypto==0.31.0', 'natsort==6.2.1', # 6.2.1 is the last version which supports Python 2. Can update once we no longer support Python 2 'netaddr==0.8.0', 'netifaces==0.10.7', 'pexpect==4.8.0', + 'poetry-semver==0.1.0', + 'prettyprinter==0.18.0', 'pyroute2==0.5.14', 'requests==2.25.0', 'sonic-platform-common', @@ -166,6 +178,7 @@ 'sonic-yang-mgmt', 'swsssdk>=2.0.1', 'tabulate==0.8.2', + 'www-authenticate==0.9.2', 'xmltodict==0.12.0', ], setup_requires= [ @@ -173,7 +186,9 @@ 'wheel' ], tests_require = [ + 'pyfakefs', 'pytest', + 'mock>=2.0.0', 'mockredispy>=2.9.3', 'sonic-config-engine' ], diff --git a/sonic-utilities-data/bash_completion.d/sonic-package-manager b/sonic-utilities-data/bash_completion.d/sonic-package-manager new file mode 100644 index 0000000000..a8a2456603 --- /dev/null +++ b/sonic-utilities-data/bash_completion.d/sonic-package-manager @@ -0,0 +1,8 @@ +_sonic_package_manager_completion() { + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) ) + return 0 +} + +complete -F _sonic_package_manager_completion -o default sonic-package-manager; diff --git a/sonic-utilities-data/bash_completion.d/spm b/sonic-utilities-data/bash_completion.d/spm new file mode 100644 index 0000000000..8931dc389c --- /dev/null +++ b/sonic-utilities-data/bash_completion.d/spm @@ -0,0 +1,8 @@ +_sonic_package_manager_completion() { + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) ) + return 0 +} + +complete -F _sonic_package_manager_completion -o default spm; diff --git a/sonic-utilities-data/templates/dump.sh.j2 b/sonic-utilities-data/templates/dump.sh.j2 new file mode 100644 index 0000000000..ebb7ed8f24 --- /dev/null +++ b/sonic-utilities-data/templates/dump.sh.j2 @@ -0,0 +1,29 @@ +#!/bin/bash + +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# + +service="{{ manifest.service.name }}" +dump_command="{{ manifest.package['debug-dump'] }}" +container_re="^${service}[0-9]*$" +{% raw %} +container_ids="$(docker ps -f name=${container_re} -f status=running --format {{.Names}})" +{% endraw %} +tmp_dir=$(mktemp -d) +tmp_dump_dir="$tmp_dir/$service" +tmp_archive=$(mktemp) + +mkdir -p "$tmp_dump_dir" + +for container_id in $container_ids; do + docker exec -t "${container_id}" ${dump_command} &> "${tmp_dump_dir}/${container_id}" +done + + +tar -C $(dirname $tmp_dump_dir) -cf $tmp_archive $service + +cat $tmp_archive +rm $tmp_archive +rm -rf $tmp_dir diff --git a/sonic-utilities-data/templates/monit.conf.j2 b/sonic-utilities-data/templates/monit.conf.j2 new file mode 100644 index 0000000000..f51efb9bee --- /dev/null +++ b/sonic-utilities-data/templates/monit.conf.j2 @@ -0,0 +1,18 @@ +############################################################################### +## +## =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +## auto-generated from {{ source }} by sonic-package-manager +## +## Monit configuration for {{ feature }} service +## process list: +{%- for process in processes %} +{%- if process.critical %} +## {{ process.name }} +{%- endif %} +{%- endfor %} +############################################################################### +{%- for process in processes %} +check program {{ feature }}|{{ process.name }} with path "/usr/bin/process_checker {{ feature }} {{ process.command }}" + if status != 0 for 5 times within 5 cycles then alert + +{% endfor %} diff --git a/sonic-utilities-data/templates/service_mgmt.sh.j2 b/sonic-utilities-data/templates/service_mgmt.sh.j2 new file mode 100644 index 0000000000..e46ba47380 --- /dev/null +++ b/sonic-utilities-data/templates/service_mgmt.sh.j2 @@ -0,0 +1,149 @@ +#!/bin/bash + +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# + +SERVICE="{{ manifest.service.name }}" +NAMESPACE_PREFIX="asic" +SONIC_DB_CLI="sonic-db-cli" +TMPDIR="/tmp/" +DEBUGLOG="${TMPDIR}/${SERVICE}.log" +[[ ! -z $DEV ]] && DEBUGLOG="${TMPDIR}/${SERVICE}-${DEV}.log" +[[ ! -z $DEV ]] && NET_NS="${NAMESPACE_PREFIX}${DEV}" # name of the network namespace +[[ ! -z $DEV ]] && SONIC_DB_CLI="${SONIC_DB_CLI} -n ${NET_NS}" + +{%- for service in manifest.service.dependent %} +{%- if service in multi_instance_services %} +MULTI_INST_DEPENDENT="${MULTI_INST_DEPENDENT} {{ service }}" +{%- else %} +DEPENDENT="${DEPENDENT} {{ service }}" +{%- endif %} +{%- endfor %} + +# Update dependent list based on other packages requirements +if [[ -f /etc/sonic/${SERVICE}_dependent ]]; then + DEPENDENT="${DEPENDENT} $(cat /etc/sonic/${SERVICE}_dependent)" +fi + +if [[ -f /etc/sonic/${SERVICE}_multi_inst_dependent ]]; then + MULTI_INST_DEPENDENT="${MULTI_INST_DEPENDENT} cat /etc/sonic/${SERVICE}_multi_inst_dependent" +fi + +function debug() +{ + /usr/bin/logger $1 + /bin/echo `date` "- $1" >> ${DEBUGLOG} +} + +function check_warm_boot() +{ + SYSTEM_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable` + SERVICE_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable` + if [[ x"$SYSTEM_WARM_START" == x"true" ]] || [[ x"$SERVICE_WARM_START" == x"true" ]]; then + WARM_BOOT="true" +{#- TODO: restore count validation for SONiC packages #} + else + WARM_BOOT="false" + fi +} + +function check_fast_boot() +{ + if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then + FAST_BOOT="true" + else + FAST_BOOT="false" + fi +} + +function start_dependent_services() { + if [[ x"$WARM_BOOT" != x"true" ]]; then + for dep in ${DEPENDENT}; do + /bin/systemctl start ${dep} + done + for dep in ${MULTI_INST_DEPENDENT}; do + if [[ ! -z $DEV ]]; then + /bin/systemctl start ${dep}@$DEV + else + /bin/systemctl start ${dep} + fi + done + fi +} + +function stop_dependent_services() { + if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then + for dep in ${DEPENDENT}; do + /bin/systemctl stop ${dep} + done + for dep in ${MULTI_INST_DEPENDENT}; do + if [[ ! -z $DEV ]]; then + /bin/systemctl stop ${dep}@$DEV + else + /bin/systemctl stop ${dep} + fi + done + fi +} + +function start() { + debug "Starting ${SERVICE}$DEV service..." + + # start service docker + /usr/bin/${SERVICE}.sh start $DEV + debug "Started ${SERVICE}$DEV service..." + +{%- if manifest.service["post-start-action"] %} + docker exec -t ${SERVICE}${DEV} {{ manifest.service["post-start-action"] }} +{%- endif %} +} + +function wait() { + start_dependent_services + + if [[ ! -z $DEV ]]; then + /usr/bin/${SERVICE}.sh wait $DEV + else + /usr/bin/${SERVICE}.sh wait + fi +} + +function stop() { + debug "Stopping ${SERVICE}$DEV service..." + +{%- if manifest.service["pre-shutdown-action"] %} + docker exec -t ${SERVICE}${DEV} {{ manifest.service["pre-shutdown-action"] }} +{%- endif %} + + # For WARM/FAST boot do not perform service stop + if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then + /usr/bin/${SERVICE}.sh stop $DEV + else + docker kill ${SERVICE}$DEV &> /dev/null || debug "Docker ${SERVICE}$DEV is not running ($?) ..." + fi + + debug "Stopped ${SERVICE}$DEV service..." + + stop_dependent_services +} + +OP=$1 +DEV=$2 + +check_warm_boot +check_fast_boot + +debug "Fast boot flag: ${SERVICE}$DEV ${FAST_BOOT}." +debug "Warm boot flag: ${SERVICE}$DEV ${WARM_BOOT}." + +case "$OP" in + start|wait|stop) + $1 + ;; + *) + echo "Usage: $0 {start|wait|stop}" + exit 1 + ;; +esac diff --git a/sonic-utilities-data/templates/sonic.service.j2 b/sonic-utilities-data/templates/sonic.service.j2 new file mode 100644 index 0000000000..72d6ab698c --- /dev/null +++ b/sonic-utilities-data/templates/sonic.service.j2 @@ -0,0 +1,39 @@ +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# +{%- set path = '/usr/local/bin' %} +{%- set multi_instance = multi_instance|default(False) %} +{%- set multi_instance_services = multi_instance_services|default([]) %} +[Unit] +Description={{ manifest.service.name }} container +{%- for service in manifest.service.requires %} +Requires={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.requisite %} +Requisite={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.after %} +After={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.before %} +Before={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +BindsTo=sonic.target +After=sonic.target +StartLimitIntervalSec=1200 +StartLimitBurst=3 + +[Service] +ExecStartPre={{path}}/{{manifest.service.name}}.sh start{% if multi_instance %} %i{% endif %} +ExecStart={{path}}/{{manifest.service.name}}.sh wait{% if multi_instance %} %i{% endif %} +ExecStop={{path}}/{{manifest.service.name}}.sh stop{% if multi_instance %} %i{% endif %} +RestartSec=30 + +{%- if not manifest.service.delayed %} +[Install] +WantedBy=sonic.target +{%- for service in manifest.service["wanted-by"] %} +WantedBy={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- endif %} diff --git a/sonic-utilities-data/templates/timer.unit.j2 b/sonic-utilities-data/templates/timer.unit.j2 new file mode 100644 index 0000000000..a757b8deb8 --- /dev/null +++ b/sonic-utilities-data/templates/timer.unit.j2 @@ -0,0 +1,15 @@ +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# +[Unit] +Description=Delays {{ manifest.service.name }} until SONiC has started +PartOf={{ manifest.service.name }}{% if multi_instance %}@%i{% endif %}.service + +[Timer] +OnUnitActiveSec=0 sec +OnBootSec=3min 30 sec +Unit={{ manifest.service.name }}{% if multi_instance %}@%i{% endif %}.service + +[Install] +WantedBy=timers.target sonic.target diff --git a/sonic_package_manager/__init__.py b/sonic_package_manager/__init__.py new file mode 100644 index 0000000000..9d8827c5e4 --- /dev/null +++ b/sonic_package_manager/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +from sonic_package_manager.manager import PackageManager + +__all__ = ['PackageManager'] diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py new file mode 100644 index 0000000000..09f0fbc0fe --- /dev/null +++ b/sonic_package_manager/constraint.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python + +""" Package version constraints module. """ + +import re +from abc import ABC +from dataclasses import dataclass, field +from typing import Dict, Union + +import semver + + +class VersionConstraint(semver.VersionConstraint, ABC): + """ Extends VersionConstraint from semver package. """ + + @staticmethod + def parse(constraint_expression: str) -> 'VersionConstraint': + """ Parse version constraint. + + Args: + constraint_expression: Expression syntax: "[[op][version]]+". + Returns: + The resulting VersionConstraint object. + """ + + return semver.parse_constraint(constraint_expression) + + +@dataclass +class ComponentConstraints: + """ ComponentConstraints is a set of components version constraints. """ + + components: Dict[str, VersionConstraint] = field(default_factory=dict) + + @staticmethod + def parse(constraints: Dict) -> 'ComponentConstraints': + """ Parse constraint from dictionary. + + Args: + constraints: dictionary with component name + as key and constraint expression as value + + Returns: + ComponentConstraints object. + + """ + + components = {component: VersionConstraint.parse(version) + for component, version in constraints.items()} + return ComponentConstraints(components) + + +@dataclass +class PackageConstraint: + """ PackageConstraint is a package version constraint. """ + + name: str + constraint: VersionConstraint + components: Dict[str, VersionConstraint] = field(default_factory=dict) + + def __str__(self): + return f'{self.name}{self.constraint}' + + @staticmethod + def from_string(constraint_expression: str) -> 'PackageConstraint': + """ Parse package constraint string which contains a package + name separated by a space with zero, one or more version constraint + expressions. A variety of version matching operators are supported + including >, <, ==, !=, ^, *. See Examples. + + Args: + constraint_expression: Expression syntax "[package name] [[op][version]]+". + + Returns: + PackageConstraint object. + + Examples: + >>> PackageConstraint.parse('syncd^1.0.0').constraint + =1.0.0,<2.0.0)> + >>> PackageConstraint.parse('swss>1.3.2 <4.2.1').constraint + 1.3.2,<4.2.1)> + >>> PackageConstraint.parse('swss').constraint + + """ + + REQUIREMENT_SPECIFIER_RE = \ + r'(?P[A-Za-z0-9_-]+)(?P.*)' + + match = re.match(REQUIREMENT_SPECIFIER_RE, constraint_expression) + if match is None: + raise ValueError(f'Invalid constraint {constraint_expression}') + groupdict = match.groupdict() + name = groupdict.get('name') + constraint = groupdict.get('constraint') or '*' + return PackageConstraint(name, VersionConstraint.parse(constraint)) + + @staticmethod + def from_dict(constraint_dict: Dict) -> 'PackageConstraint': + """ Parse package constraint information from dictionary. E.g: + + { + "name": "swss", + "version": "^1.0.0", + "componenets": { + "libswsscommon": "^1.0.0" + } + } + + Args: + constraint_dict: Dictionary of constraint infromation. + + Returns: + PackageConstraint object. + """ + + name = constraint_dict['name'] + version = VersionConstraint.parse(constraint_dict.get('version') or '*') + components = {component: VersionConstraint.parse(version) + for component, version in constraint_dict.get('components', {}).items()} + return PackageConstraint(name, version, components) + + @staticmethod + def parse(constraint: Union[str, Dict]) -> 'PackageConstraint': + """ Parse constraint from string expression or dictionary. + + Args: + constraint: string or dictionary. Check from_str() and from_dict() methods. + + Returns: + PackageConstraint object. + + """ + + if type(constraint) is str: + return PackageConstraint.from_string(constraint) + elif type(constraint) is dict: + return PackageConstraint.from_dict(constraint) + else: + raise ValueError('Input argument should be either str or dict') + diff --git a/sonic_package_manager/database.py b/sonic_package_manager/database.py new file mode 100644 index 0000000000..6c1cec5c07 --- /dev/null +++ b/sonic_package_manager/database.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python + +""" Repository Database interface module. """ +import json +import os +from dataclasses import dataclass, replace +from typing import Optional, Dict, Callable + +from sonic_package_manager.errors import PackageManagerError, PackageNotFoundError, PackageAlreadyExistsError +from sonic_package_manager.version import Version + +BASE_LIBRARY_PATH = '/var/lib/sonic-package-manager/' +PACKAGE_MANAGER_DB_FILE_PATH = os.path.join(BASE_LIBRARY_PATH, 'packages.json') +PACKAGE_MANAGER_LOCK_FILE = os.path.join(BASE_LIBRARY_PATH, '.lock') + + +@dataclass(order=True) +class PackageEntry: + """ Package database single entry object. + + Attributes: + name: Name of the package + repository: Default repository to pull package from. + description: Package description or None if package does not + provide a description. + default_reference: Default reference (tag or digest) or None + if default reference is not provided. + version: Installed version of the package or None if + package is not installed. + installed: Boolean flag whether the package is installed. + built_in: Boolean flag whether the package is built in. + image_id: Image ID for this package or None if package + is not installed. + """ + + name: str + repository: Optional[str] + description: Optional[str] = None + default_reference: Optional[str] = None + version: Optional[Version] = None + installed: bool = False + built_in: bool = False + image_id: Optional[str] = None + + +def package_from_dict(name: str, package_info: Dict) -> PackageEntry: + """ Parse dictionary into PackageEntry object.""" + + repository = package_info.get('repository') + description = package_info.get('description') + default_reference = package_info.get('default-reference') + version = package_info.get('installed-version') + if version: + version = Version.parse(version) + installed = package_info.get('installed', False) + built_in = package_info.get('built-in', False) + image_id = package_info.get('image-id') + + return PackageEntry(name, repository, description, + default_reference, version, installed, + built_in, image_id) + + +def package_to_dict(package: PackageEntry) -> Dict: + """ Serialize package into dictionary. """ + + return { + 'repository': package.repository, + 'description': package.description, + 'default-reference': package.default_reference, + 'installed-version': None if package.version is None else str(package.version), + 'installed': package.installed, + 'built-in': package.built_in, + 'image-id': package.image_id, + } + + +class PackageDatabase: + """ An interface to SONiC repository database """ + + def __init__(self, + database: Dict[str, PackageEntry], + on_save: Optional[Callable] = None): + """ Initialize PackageDatabase. + + Args: + database: Database dictionary + on_save: Optional callback to execute on commit() + """ + + self._database = database + self._on_save = on_save + + def add_package(self, + name: str, + repository: str, + description: Optional[str] = None, + default_reference: Optional[str] = None): + """ Adds a new package entry in database. + + Args: + name: Package name. + repository: Repository URL. + description: Description string. + default_reference: Default version string. + + Raises: + PackageAlreadyExistsError: if package already exists in database. + """ + + if self.has_package(name): + raise PackageAlreadyExistsError(name) + + package = PackageEntry(name, repository, description, default_reference) + self._database[name] = package + + def remove_package(self, name: str): + """ Removes package entry from database. + + Args: + name: repository name. + Raises: + PackageNotFoundError: Raises when package with the given name does not exist + in the database. + """ + + pkg = self.get_package(name) + + if pkg.built_in: + raise PackageManagerError(f'Package {name} is built-in, cannot remove it') + + if pkg.installed: + raise PackageManagerError(f'Package {name} is installed, uninstall it first') + + self._database.pop(name) + + def update_package(self, pkg: PackageEntry): + """ Modify repository in the database. + + Args: + pkg: Repository object. + Raises: + PackageManagerError: Raises when repository with the given name does not exist + in the database. + """ + + name = pkg.name + + if not self.has_package(name): + raise PackageNotFoundError(name) + + self._database[name] = pkg + + def get_package(self, name: str) -> PackageEntry: + """ Return a package referenced by name. + If the package is not found PackageNotFoundError is thrown. + + Args: + name: Package name. + Returns: + PackageInfo object. + Raises: + PackageNotFoundError: When package called name was not found. + """ + + try: + pkg = self._database[name] + except KeyError: + raise PackageNotFoundError(name) + + return replace(pkg) + + def has_package(self, name: str) -> bool: + """ Checks if the database contains an entry for a package. + called name. Returns True if the package exists, otherwise False. + + Args: + name: Package name. + Returns: + True if the package exists, otherwise False. + """ + + try: + self.get_package(name) + return True + except PackageNotFoundError: + return False + + def __iter__(self): + """ Iterates over packages in the database. + + Yields: + PackageInfo object. + """ + + for name, _ in self._database.items(): + yield self.get_package(name) + + @staticmethod + def from_file(db_file=PACKAGE_MANAGER_DB_FILE_PATH) -> 'PackageDatabase': + """ Read database content from file. """ + + def on_save(database): + with open(db_file, 'w') as db: + db_content = {} + for name, package in database.items(): + db_content[name] = package_to_dict(package) + json.dump(db_content, db, indent=4) + + database = {} + with open(db_file) as db: + db_content = json.load(db) + for key in db_content: + package = package_from_dict(key, db_content[key]) + database[key] = package + return PackageDatabase(database, on_save) + + def commit(self): + """ Save database content to file. """ + + if self._on_save: + self._on_save(self._database) diff --git a/sonic_package_manager/dockerapi.py b/sonic_package_manager/dockerapi.py new file mode 100644 index 0000000000..926600d0bc --- /dev/null +++ b/sonic_package_manager/dockerapi.py @@ -0,0 +1,226 @@ +#!/usr/bin/evn python + +""" Module provides Docker interface. """ + +import contextlib +import io +import tarfile +import re +from typing import Optional + +from sonic_package_manager.logger import log +from sonic_package_manager.progress import ProgressManager + + +def is_digest(ref: str): + return ref.startswith('sha256:') + + +def bytes_to_mb(bytes): + return bytes / 1024 / 1024 + + +def get_id(line): + return line['id'] + + +def get_status(line): + return line['status'] + + +def get_progress(line): + progress = line['progressDetail'] + current = bytes_to_mb(progress['current']) + total = bytes_to_mb(progress['total']) + return current, total + + +def process_progress(progress_manager, line): + try: + status = get_status(line) + id = get_id(line) + current, total = get_progress(line) + + if id not in progress_manager: + progress_manager.new(id, + total=total, + unit='Mb', + desc=f'{status} {id}') + pbar = progress_manager.get(id) + + # Complete status + if 'complete' in status: + pbar.desc = f'{status} {id}' + pbar.update(pbar.total) + return + + # Status changed + if status not in pbar.desc: + pbar.desc = f'{status} {id}' + pbar.total = total + pbar.count = 0 + + pbar.update(current - pbar.count) + except KeyError: + # not a progress line + return + + +def get_repository_from_image(image): + """ Returns the first RepoTag repository + found in image. """ + + repotags = image.attrs['RepoTags'] + for repotag in repotags: + repository, tag = repotag.split(':') + return repository + + +class DockerApi: + """ DockerApi provides a set of methods - + wrappers around docker client methods """ + + def __init__(self, + client, + progress_manager: Optional[ProgressManager] = None): + self.client = client + self.progress_manager = progress_manager + + def pull(self, repository: str, + reference: Optional[str] = None): + """ Docker 'pull' command. + Args: + repository: repository to pull + reference: tag or digest + """ + + log.debug(f'pulling image from {repository} reference={reference}') + + api = self.client.api + progress_manager = self.progress_manager + + digest = None + + with progress_manager or contextlib.nullcontext(): + for line in api.pull(repository, + reference, + stream=True, + decode=True): + log.debug(f'pull status: {line}') + + status = get_status(line) + + # Record pulled digest + digest_match = re.match(r'Digest: (?P.*)', status) + if digest_match: + digest = digest_match.groupdict()['sha'] + + if progress_manager: + process_progress(progress_manager, line) + + log.debug(f'Digest: {digest}') + log.debug(f'image from {repository} reference={reference} pulled successfully') + + return self.get_image(f'{repository}@{digest}') + + def load(self, imgpath: str): + """ Docker 'load' command. + Args: + + """ + + log.debug(f'loading image from {imgpath}') + + api = self.client.api + progress_manager = self.progress_manager + + imageid = None + repotag = None + + with progress_manager or contextlib.nullcontext(): + with open(imgpath, 'rb') as imagefile: + for line in api.load_image(imagefile, quiet=False): + log.debug(f'pull status: {line}') + + if progress_manager: + process_progress(progress_manager, line) + + if 'stream' not in line: + continue + + stream = line['stream'] + repotag_match = re.match(r'Loaded image: (?P.*)\n', stream) + if repotag_match: + repotag = repotag_match.groupdict()['repotag'] + imageid_match = re.match(r'Loaded image ID: sha256:(?P.*)\n', stream) + if imageid_match: + imageid = imageid_match.groupdict()['id'] + + imagename = repotag if repotag else imageid + log.debug(f'Loaded image {imagename}') + + return self.get_image(imagename) + + def rmi(self, image: str, **kwargs): + """ Docker 'rmi -f' command. """ + + log.debug(f'removing image {image} kwargs={kwargs}') + + self.client.images.remove(image, **kwargs) + + log.debug(f'image {image} removed successfully') + + def tag(self, image: str, repotag: str, **kwargs): + """ Docker 'tag' command """ + + log.debug(f'tagging image {image} {repotag} kwargs={kwargs}') + + img = self.client.images.get(image) + img.tag(repotag, **kwargs) + + log.debug(f'image {image} tagged {repotag} successfully') + + def rm(self, container: str, **kwargs): + """ Docker 'rm' command. """ + + self.client.containers.get(container).remove(**kwargs) + log.debug(f'removed container {container}') + + def ps(self, **kwargs): + """ Docker 'ps' command. """ + + return self.client.containers.list(**kwargs) + + def labels(self, image: str): + """ Returns a list of labels associated with image. """ + + log.debug(f'inspecting image labels {image}') + + labels = self.client.images.get(image).labels + + log.debug(f'image {image} labels successfully: {labels}') + return labels + + def get_image(self, name: str): + return self.client.images.get(name) + + def extract(self, image, src_path: str, dst_path: str): + """ Copy src_path from the docker image to host dst_path. """ + + buf = bytes() + + container = self.client.containers.create(image) + try: + bits, _ = container.get_archive(src_path) + for chunk in bits: + buf += chunk + finally: + container.remove(force=True) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + for member in tar: + if dst_path.endswith('/'): + tar.extract(member, dst_path) + else: + member.name = dst_path + tar.extract(member, dst_path) diff --git a/sonic_package_manager/errors.py b/sonic_package_manager/errors.py new file mode 100644 index 0000000000..17279c52c4 --- /dev/null +++ b/sonic_package_manager/errors.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python + +""" SONiC Package Manager exceptions are defined in this module. """ + +from dataclasses import dataclass +from typing import Optional + +from sonic_package_manager.constraint import PackageConstraint, VersionConstraint +from sonic_package_manager.version import Version + + +class PackageManagerError(Exception): + """ Base class for exceptions generated by SONiC package manager """ + + pass + + +class ManifestError(Exception): + """ Class for manifest validate failures. """ + + pass + + +class MetadataError(Exception): + """ Class for metadata failures. """ + + pass + + +@dataclass +class PackageNotFoundError(PackageManagerError): + """ Repository not found in repository database exception """ + + name: str + + def __str__(self): + return f'Package {self.name} is not found in packages database' + + +@dataclass +class PackageAlreadyExistsError(PackageManagerError): + """ Package already exists in the packages database exception. """ + + name: str + + def __str__(self): + return f'Package {self.name} already exists in packages database' + + +class PackageInstallationError(PackageManagerError): + """ Exception for package installation error. """ + + pass + + +class PackageUninstallationError(PackageManagerError): + """ Exception for package installation error. """ + + pass + + +class PackageUpgradeError(PackageManagerError): + """ Exception for package upgrade error. """ + + pass + + +@dataclass +class PackageSonicRequirementError(PackageInstallationError): + """ Exception for installation errors, when SONiC version requirement is not met. """ + + name: str + component: str + constraint: PackageConstraint + installed_ver: Optional[Version] = None + + def __str__(self): + if self.installed_ver is not None: + return (f'Package {self.name} requires base OS component {self.component} version {self.constraint} ' + f'while the installed version is {self.installed_ver}') + return (f'Package {self.name} requires base OS component {self.component} version {self.constraint} ' + f'but it is not present int base OS image') + + +@dataclass +class PackageDependencyError(PackageInstallationError): + """ Exception class for installation errors related to missing dependency. """ + + name: str + constraint: PackageConstraint + installed_ver: Optional[Version] = None + + def __str__(self): + if self.installed_ver: + return (f'Package {self.name} requires {self.constraint} ' + f'but version {self.installed_ver} is installed') + return f'Package {self.name} requires {self.constraint} but it is not installed' + + +@dataclass +class PackageComponentDependencyError(PackageInstallationError): + """ Exception class for installation error caused by component + version dependency. """ + + name: str + dependency: str + component: str + constraint: VersionConstraint + installed_ver: Optional[Version] = None + + def __str__(self): + if self.installed_ver: + return (f'Package {self.name} requires {self.component} {self.constraint} ' + f'in package {self.dependency} but version {self.installed_ver} is installed') + return (f'Package {self.name} requires {self.component} {self.constraint} ' + f'in package {self.dependency} but it is not installed') + + +@dataclass +class PackageConflictError(PackageInstallationError): + """ Exception class for installation errors related to missing dependency. """ + + name: str + constraint: PackageConstraint + installed_ver: Version + + def __str__(self): + return (f'Package {self.name} conflicts with {self.constraint} but ' + f'version {self.installed_ver} is installed') + + +@dataclass +class PackageComponentConflictError(PackageInstallationError): + """ Exception class for installation error caused by component + version conflict. """ + + name: str + dependency: str + component: str + constraint: VersionConstraint + installed_ver: Version + + def __str__(self): + return (f'Package {self.name} conflicts with {self.component} {self.constraint} ' + f'in package {self.dependency} but version {self.installed_ver} is installed') + diff --git a/sonic_package_manager/logger.py b/sonic_package_manager/logger.py new file mode 100644 index 0000000000..3d5e06d35f --- /dev/null +++ b/sonic_package_manager/logger.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +""" Logger for sonic-package-manager. """ + +import logging.handlers + +import click_log + + +class Formatter(click_log.ColorFormatter): + """ Click logging formatter. """ + + colors = { + 'error': dict(fg='red'), + 'exception': dict(fg='red'), + 'critical': dict(fg='red'), + 'debug': dict(fg='blue', bold=True), + 'warning': dict(fg='yellow'), + } + + +log = logging.getLogger("sonic-package-manager") +log.setLevel(logging.INFO) + +click_handler = click_log.ClickHandler() +click_handler.formatter = Formatter() + +log.addHandler(click_handler) +log.addHandler(logging.handlers.SysLogHandler()) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py new file mode 100644 index 0000000000..dcc048079c --- /dev/null +++ b/sonic_package_manager/main.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python + +import functools +import json +import os +import sys +import typing + +import click +import click_log +import tabulate +from natsort import natsorted + +from sonic_package_manager.database import PackageEntry, PackageDatabase +from sonic_package_manager.errors import PackageManagerError +from sonic_package_manager.logger import log +from sonic_package_manager.manager import PackageManager + +BULLET_UC = '\u2022' + + +def exit_cli(*args, **kwargs): + """ Print a message and exit with rc 1. """ + + click.secho(*args, **kwargs) + sys.exit(1) + + +def show_help(ctx): + """ Show help message and exit process successfully. """ + + click.echo(ctx.get_help()) + ctx.exit(0) + + +def root_privileges_required(func: typing.Callable) -> typing.Callable: + """ Decorates a function, so that the function is invoked + only if the user is root. """ + + @functools.wraps(func) + def wrapped_function(*args, **kwargs): + """ Wrapper around func. """ + + if os.geteuid() != 0: + exit_cli('Root privileges required for this operation', fg='red') + + return func(*args, **kwargs) + + return wrapped_function + + +def add_options(options): + """ Decorator to append options from + input list to command. """ + + def _add_options(func): + for option in reversed(options): + func = option(func) + return func + + return _add_options + + +class MutuallyExclusiveOption(click.Option): + """ This options type is extended with 'mutually_exclusive' + parameter which makes CLI to check if several options are now + used together in single command. """ + + def __init__(self, *args, **kwargs): + self.mutually_exclusive = set(kwargs.pop('mutually_exclusive', [])) + help_string = kwargs.get('help', '') + if self.mutually_exclusive: + ex_str = ', '.join(self.mutually_exclusive) + kwargs['help'] = f'{help_string} ' \ + f'NOTE: This argument is mutually ' \ + f'exclusive with arguments: [{ex_str}].' + super().__init__(*args, **kwargs) + + def handle_parse_result(self, ctx, opts, args): + if self.name in opts and opts[self.name] is not None: + for opt_name in self.mutually_exclusive.intersection(opts): + if opts[opt_name] is None: + continue + + raise click.UsageError(f'Illegal usage: {self.name} is mutually ' + f'exclusive with arguments ' + f'{", ".join(self.mutually_exclusive)}.') + + return super().handle_parse_result(ctx, opts, args) + + +PACKAGE_SOURCE_OPTIONS = [ + click.option('--from-repository', + help='Install package directly from image registry repository', + cls=MutuallyExclusiveOption, + mutually_exclusive=['from_tarball', 'package_expr']), + click.option('--from-tarball', + type=click.Path(exists=True, + readable=True, + file_okay=True, + dir_okay=False), + help='Install package from saved image tarball', + cls=MutuallyExclusiveOption, + mutually_exclusive=['from_repository', 'package_expr']), + click.argument('package-expr', + type=str, + required=False) +] + + +PACKAGE_COMMON_INSTALL_OPTIONS = [ + click.option('--skip-cli-plugin-installation', + is_flag=True, + help='Do not install CLI plugins provided by the package ' + 'on the host OS. NOTE: In case when package /cli/mandatory ' + 'field is set to True this option will fail the installation.'), +] + + +PACKAGE_COMMON_OPERATION_OPTIONS = [ + click.option('-f', '--force', + is_flag=True, + help='Force operation by ignoring failures'), + click.option('-y', '--yes', + is_flag=True, + help='Automatically answer yes on prompts'), + click_log.simple_verbosity_option(log), +] + + +def get_package_status(package: PackageEntry): + """ Returns the installation status message for package. """ + + if package.built_in: + return 'Built-In' + elif package.installed: + return 'Installed' + else: + return 'Not Installed' + + +@click.group() +@click.pass_context +def cli(ctx): + """ SONiC Package Manager """ + + ctx.obj = PackageManager.get_manager() + + +@cli.group() +@click.pass_context +def repository(ctx): + """ Repository management commands """ + + pass + + +@cli.group() +@click.pass_context +def show(ctx): + """ Package manager show commands """ + + pass + + +@show.group() +@click.pass_context +def package(ctx): + """ Package show commands """ + + pass + + +@cli.command() +@click.pass_context +def list(ctx): + """ List available repositories """ + + table_header = ['Name', 'Repository', 'Description', 'Version', 'Status'] + table_body = [] + + manager: PackageManager = ctx.obj + + try: + for package in natsorted(manager.database): + repository = package.repository or 'N/A' + version = package.version or 'N/A' + description = package.description or 'N/A' + status = get_package_status(package) + + table_body.append([ + package.name, + repository, + description, + version, + status + ]) + + click.echo(tabulate.tabulate(table_body, table_header)) + except PackageManagerError as err: + exit_cli(f'Failed to list repositories: {err}', fg='red') + + +@package.command() +@add_options(PACKAGE_SOURCE_OPTIONS) +@click.pass_context +def manifest(ctx, + package_expr, + from_repository, + from_tarball): + """ Print package manifest content """ + + manager: PackageManager = ctx.obj + + try: + source = manager.get_package_source(package_expr, + from_repository, + from_tarball) + package = source.get_package() + click.echo(json.dumps(package.manifest.unmarshal(), indent=4)) + except Exception as err: + exit_cli(f'Failed to print manifest: {err}', fg='red') + + +@package.command() +@click.argument('name') +@click.option('--all', is_flag=True, help='Show all available tags in repository') +@click.option('--plain', is_flag=True, help='Plain output') +@click.pass_context +def versions(ctx, name, all, plain): + """ Print available versions """ + + try: + manager: PackageManager = ctx.obj + versions = manager.get_package_available_versions(name, all) + for version in versions: + if not plain: + click.secho(f'{BULLET_UC} ', bold=True, fg='green', nl=False) + click.secho(f'{version}') + except Exception as err: + exit_cli(f'Failed to get package versions for {name}: {err}', fg='red') + + +@package.command() +@add_options(PACKAGE_SOURCE_OPTIONS) +@click.pass_context +def changelog(ctx, + package_expr, + from_repository, + from_tarball): + """ Print package changelog """ + + manager: PackageManager = ctx.obj + + try: + source = manager.get_package_source(package_expr, + from_repository, + from_tarball) + package = source.get_package() + changelog = package.manifest['package']['changelog'] + + if not changelog: + raise PackageManagerError(f'No changelog for package {package.name}') + + for version, entry in changelog.items(): + author = entry.get('author') or 'N/A' + email = entry.get('email') or 'N/A' + changes = entry.get('changes') or [] + date = entry.get('date') or 'N/A' + click.secho(f'{version}:\n', fg='green', bold=True) + for line in changes: + click.secho(f' {BULLET_UC} {line}', bold=True) + click.secho(f'\n {author} ' + f'({email}) {date}', fg='green', bold=True) + click.secho('') + + except Exception as err: + exit_cli(f'Failed to print package changelog: {err}', fg='red') + + +@repository.command() +@click.argument('name', type=str) +@click.argument('repository', type=str) +@click.option('--default-reference', type=str) +@click.option('--description', type=str) +@click.pass_context +@root_privileges_required +def add(ctx, name, repository, default_reference, description): + """ Add a new repository to database. """ + + manager: PackageManager = ctx.obj + + try: + manager.add_repository(name, + repository, + description=description, + default_reference=default_reference) + except Exception as err: + exit_cli(f'Failed to add repository {name}: {err}', fg='red') + + +@repository.command() +@click.argument("name") +@click.pass_context +@root_privileges_required +def remove(ctx, name): + """ Remove package from database. """ + + manager: PackageManager = ctx.obj + + try: + manager.remove_repository(name) + except Exception as err: + exit_cli(f'Failed to remove repository {name}: {err}', fg='red') + + +@cli.command() +@click.option('--enable', + is_flag=True, + help='Set the default state of the feature to enabled ' + 'and enable feature right after installation. ' + 'NOTE: user needs to execute "config save -y" to make ' + 'this setting persistent') +@click.option('--default-owner', + type=click.Choice(['local', 'kube']), + default='local', + help='Default owner configuration setting for a feature') +@add_options(PACKAGE_SOURCE_OPTIONS) +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@add_options(PACKAGE_COMMON_INSTALL_OPTIONS) +@click.pass_context +@root_privileges_required +def install(ctx, + package_expr, + from_repository, + from_tarball, + force, + yes, + enable, + default_owner, + skip_cli_plugin_installation): + """ Install package """ + + manager: PackageManager = ctx.obj + + package_source = package_expr or from_repository or from_tarball + + if not yes and not force: + click.confirm(f'{package_source} is going to be installed, ' + f'continue?', abort=True, show_default=True) + + install_opts = { + 'force': force, + 'enable': enable, + 'default_owner': default_owner, + 'skip_cli_plugin_installation': skip_cli_plugin_installation, + } + + try: + manager.install(package_expr, + from_repository, + from_tarball, + **install_opts) + except Exception as err: + exit_cli(f'Failed to install {package_source}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@add_options(PACKAGE_SOURCE_OPTIONS) +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@add_options(PACKAGE_COMMON_INSTALL_OPTIONS) +@click.pass_context +@root_privileges_required +def upgrade(ctx, + package_expr, + from_repository, + from_tarball, + force, + yes): + """ Upgrade package """ + + manager: PackageManager = ctx.obj + + package_source = package_expr or from_repository or from_tarball + + if not yes and not force: + click.confirm(f'Package is going to be upgraded with {package_source}, ' + f'continue?', abort=True, show_default=True) + + upgrade_opts = { + 'force': force, + 'skip_cli_plugin_installation': skip_cli_plugin_installation, + } + + try: + manager.upgrade(package_expr, + from_repository, + from_tarball, + **upgrade_opts) + except Exception as err: + exit_cli(f'Failed to upgrade {package_source}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@click.argument('name') +@click.pass_context +@root_privileges_required +def uninstall(ctx, name, force, yes): + """ Uninstall package """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm(f'Package {name} is going to be uninstalled, ' + f'continue?', abort=True, show_default=True) + + try: + manager.uninstall(name, force) + except Exception as err: + exit_cli(f'Failed to uninstall package {name}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@click.option('--dockerd-socket', type=click.Path()) +@click.argument('database', type=click.Path()) +@click.pass_context +@root_privileges_required +def migrate(ctx, database, force, yes, dockerd_socket): + """ Migrate packages from the given database file """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm('Continue with package migration?', abort=True, show_default=True) + + try: + manager.migrate_packages(PackageDatabase.from_file(database), dockerd_socket) + except Exception as err: + exit_cli(f'Failed to migrate packages {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +if __name__ == "__main__": + cli() diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py new file mode 100644 index 0000000000..f1643a1846 --- /dev/null +++ b/sonic_package_manager/manager.py @@ -0,0 +1,896 @@ +#!/usr/bin/env python +import contextlib +import functools +import os +import pkgutil +import tempfile +from typing import Any, Iterable, Callable, Dict, Optional + +import docker +import filelock +from sonic_py_common import device_info + +from sonic_package_manager import utils +from sonic_package_manager.constraint import ( + VersionConstraint, + PackageConstraint +) +from sonic_package_manager.database import ( + PACKAGE_MANAGER_LOCK_FILE, + PackageDatabase +) +from sonic_package_manager.dockerapi import DockerApi +from sonic_package_manager.errors import ( + PackageManagerError, + PackageDependencyError, + PackageComponentDependencyError, + PackageConflictError, + PackageComponentConflictError, + PackageInstallationError, + PackageSonicRequirementError, + PackageUninstallationError, + PackageUpgradeError +) +from sonic_package_manager.logger import log +from sonic_package_manager.metadata import MetadataResolver +from sonic_package_manager.package import Package +from sonic_package_manager.progress import ProgressManager +from sonic_package_manager.reference import PackageReference +from sonic_package_manager.registry import RegistryResolver +from sonic_package_manager.service_creator.creator import ServiceCreator, run_command +from sonic_package_manager.service_creator.feature import FeatureRegistry +from sonic_package_manager.service_creator.sonic_db import SonicDB +from sonic_package_manager.service_creator.utils import in_chroot +from sonic_package_manager.source import ( + PackageSource, + LocalSource, + RegistrySource, + TarballSource +) +from sonic_package_manager.utils import DockerReference +from sonic_package_manager.version import ( + Version, + VersionRange, + version_to_tag, tag_to_version +) + + +@contextlib.contextmanager +def failure_ignore(ignore: bool): + """ Ignores failures based on parameter passed. """ + + try: + yield + except Exception as err: + if ignore: + log.warning(f'ignoring error {err}') + else: + raise + + +def under_lock(func: Callable) -> Callable: + """ Execute operations under lock. """ + + @functools.wraps(func) + def wrapped_function(*args, **kwargs): + self = args[0] + with self.lock: + return func(*args, **kwargs) + + return wrapped_function + + +def rollback_wrapper(func, *args, **kwargs): + """ Used in rollback callbacks to ignore failure + but proceed with rollback. Error will be printed + but not fail the whole procedure of rollback. """ + + @functools.wraps(func) + def wrapper(): + try: + func(*args, **kwargs) + except Exception as err: + log.error(f'failed in rollback: {err}') + + return wrapper + + +def package_constraint_to_reference(constraint: PackageConstraint) -> PackageReference: + package_name, version_constraint = constraint.name, constraint.constraint + # Allow only specific version for now. + # Later we can improve package manager to support + # installing packages using expressions like 'package>1.0.0' + if version_constraint == VersionRange(): # empty range means any version + return PackageReference(package_name, None) + if not isinstance(version_constraint, Version): + raise PackageManagerError(f'Can only install specific version. ' + f'Use only following expression "{package_name}==" ' + f'to install specific version') + return PackageReference(package_name, version_to_tag(version_constraint)) + + +def parse_reference_expression(expression): + try: + return package_constraint_to_reference(PackageConstraint.parse(expression)) + except ValueError: + # if we failed to parse the expression as constraint expression + # we will try to parse it as reference + return PackageReference.parse(expression) + + +def validate_package_base_os_constraints(package: Package, sonic_version_info: Dict[str, str]): + """ Verify that all dependencies on base OS components are met. + Args: + package: Package to check constraints for. + sonic_version_info: SONiC components version information. + Raises: + PackageSonicRequirementError: in case dependency is not satisfied. + """ + + base_os_constraints = package.manifest['package']['base-os'].components + for component, constraint in base_os_constraints.items(): + if component not in sonic_version_info: + raise PackageSonicRequirementError(package.name, component, constraint) + + version = Version.parse(sonic_version_info[component]) + + if not constraint.allows_all(version): + raise PackageSonicRequirementError(package.name, component, constraint, version) + + +def validate_package_tree(packages: Dict[str, Package]): + """ Verify that all dependencies are met in all packages passed to this function. + Args: + packages: list of packages to check + Raises: + PackageDependencyError: if dependency is missing + PackageConflictError: if there is a conflict between packages + """ + + for name, package in packages.items(): + log.debug(f'checking dependencies for {name}') + for dependency in package.manifest['package']['depends']: + dependency_package = packages.get(dependency.name) + if dependency_package is None: + raise PackageDependencyError(package.name, dependency) + + installed_version = dependency_package.version + log.debug(f'dependency package is installed {dependency.name}: {installed_version}') + if not dependency.constraint.allows_all(installed_version): + raise PackageDependencyError(package.name, dependency, installed_version) + + dependency_components = dependency.components + if not dependency_components: + dependency_components = {} + for component, version in package.components.items(): + implicit_constraint = VersionConstraint.parse(f'^{version.major}.{version.minor}.0') + dependency_components[component] = implicit_constraint + + for component, constraint in dependency_components.items(): + if component not in dependency_package.components: + raise PackageComponentDependencyError(package.name, dependency, + component, constraint) + + component_version = dependency_package.components[component] + log.debug(f'dependency package {dependency.name}: ' + f'component {component} version is {component_version}') + + if not constraint.allows_all(component_version): + raise PackageComponentDependencyError(package.name, dependency, component, + constraint, component_version) + + log.debug(f'checking conflicts for {name}') + for conflict in package.manifest['package']['breaks']: + conflicting_package = packages.get(conflict.name) + if conflicting_package is None: + continue + + installed_version = conflicting_package.version + log.debug(f'conflicting package is installed {conflict.name}: {installed_version}') + if conflict.constraint.allows_all(installed_version): + raise PackageConflictError(package.name, conflict, installed_version) + + for component, constraint in conflicting_package.components.items(): + if component not in conflicting_package.components: + continue + + component_version = conflicting_package.components[component] + log.debug(f'conflicting package {dependency.name}: ' + f'component {component} version is {component_version}') + + if constraint.allows_all(component_version): + raise PackageComponentConflictError(package.name, dependency, component, + constraint, component_version) + + +def validate_package_cli_can_be_skipped(package: Package, skip: bool): + """ Checks whether package CLI installation can be skipped. + + Args: + package: Package to validate + skip: Whether to skip installing CLI + + Raises: + PackageManagerError + + """ + + if package.manifest['cli']['mandatory'] and skip: + raise PackageManagerError(f'CLI is mandatory for package {package.name} ' + f'but it was requested to be not installed') + elif skip: + log.warning(f'Package {package.name} CLI plugin will not be installed') + + +class PackageManager: + """ SONiC Package Manager. This class provides public API + for sonic_package_manager python library. It has functionality + for installing, uninstalling, updating SONiC packages as well as + retrieving information about the packages from different sources. """ + + def __init__(self, + docker_api: DockerApi, + registry_resolver: RegistryResolver, + database: PackageDatabase, + metadata_resolver: MetadataResolver, + service_creator: ServiceCreator, + device_information: Any, + lock: filelock.FileLock): + """ Initialize PackageManager. """ + + self.lock = lock + self.docker = docker_api + self.registry_resolver = registry_resolver + self.database = database + self.metadata_resolver = metadata_resolver + self.service_creator = service_creator + self.feature_registry = service_creator.feature_registry + self.is_multi_npu = device_information.is_multi_npu() + self.num_npus = device_information.get_num_npus() + self.version_info = device_information.get_sonic_version_info() + + @under_lock + def add_repository(self, *args, **kwargs): + """ Add repository to package database + and commit database content. + + Args: + args: Arguments to pass to PackageDatabase.add_package + kwargs: Keyword arguments to pass to PackageDatabase.add_package + """ + + self.database.add_package(*args, **kwargs) + self.database.commit() + + @under_lock + def remove_repository(self, name: str): + """ Remove repository from package database + and commit database content. + + Args: + name: package name + """ + + self.database.remove_package(name) + self.database.commit() + + @under_lock + def install(self, + expression: Optional[str] = None, + repotag: Optional[str] = None, + tarball: Optional[str] = None, + **kwargs): + """ Install SONiC Package from either an expression representing + the package and its version, repository and tag or digest in + same format as "docker pulL" accepts or an image tarball path. + + Args: + expression: SONiC Package reference expression + repotag: Install from REPO[:TAG][@DIGEST] + tarball: Install from tarball, path to tarball file + kwargs: Install options for self.install_from_source + Raises: + PackageManagerError + """ + + source = self.get_package_source(expression, repotag, tarball) + self.install_from_source(source, **kwargs) + + @under_lock + def install_from_source(self, + source: PackageSource, + force=False, + enable=False, + default_owner='local', + skip_cli_plugin_installation=False): + """ Install SONiC Package from source represented by PackageSource. + This method contains the logic of package installation. + + Args: + source: SONiC Package source. + force: Force the installation. + enable: If True the installed feature package will be enabled. + default_owner: Owner of the installed package. + skip_cli_plugin_installation: Skip CLI plugin installation. + Raises: + PackageManagerError + """ + + package = source.get_package() + name = package.name + + with failure_ignore(force): + if self.is_installed(name): + raise PackageInstallationError(f'{name} is already installed') + + version = package.manifest['package']['version'] + feature_state = 'enabled' if enable else 'disabled' + installed_packages = self._get_installed_packages_and(package) + + with failure_ignore(force): + validate_package_base_os_constraints(package, self.version_info) + validate_package_tree(installed_packages) + validate_package_cli_can_be_skipped(package, skip_cli_plugin_installation) + + # After all checks are passed we proceed to actual installation + + # When installing package from a tarball or directly from registry + # package name may not be in database. + if not self.database.has_package(package.name): + self.database.add_package(package.name, package.repository) + + try: + with contextlib.ExitStack() as exit_stack: + source.install(package) + exit_stack.callback(rollback_wrapper(source.uninstall, package)) + + self.service_creator.create(package, state=feature_state, owner=default_owner) + exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) + + if not skip_cli_plugin_installation: + self._install_cli_plugins(package) + exit_stack.callback(rollback_wrapper(self._uninstall_cli_plugins, package)) + + exit_stack.pop_all() + except Exception as err: + raise PackageInstallationError(f'Failed to install {package.name}: {err}') + except KeyboardInterrupt: + raise + + package.entry.installed = True + package.entry.version = version + self.database.update_package(package.entry) + self.database.commit() + + @under_lock + def uninstall(self, name: str, force=False): + """ Uninstall SONiC Package referenced by name. The uninstallation + can be forced if force argument is True. + + Args: + name: SONiC Package name. + force: Force the installation. + Raises: + PackageManagerError + """ + + with failure_ignore(force): + if not self.is_installed(name): + raise PackageUninstallationError(f'{name} is not installed') + + package = self.get_installed_package(name) + service_name = package.manifest['service']['name'] + + with failure_ignore(force): + if self.feature_registry.is_feature_enabled(service_name): + raise PackageUninstallationError( + f'{service_name} is enabled. Disable the feature first') + + if package.built_in: + raise PackageUninstallationError( + f'Cannot uninstall built-in package {package.name}') + + installed_packages = self._get_installed_packages_except(package) + + with failure_ignore(force): + validate_package_tree(installed_packages) + + # After all checks are passed we proceed to actual uninstallation + + try: + self._uninstall_cli_plugins(package) + self.service_creator.remove(package) + + # Clean containers based on this image + containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) + for container in containers: + self.docker.rm(container.id, force=True) + + self.docker.rmi(package.image_id, force=True) + package.entry.image_id = None + except Exception as err: + raise PackageUninstallationError(f'Failed to uninstall {package.name}: {err}') + + package.entry.installed = False + package.entry.version = None + self.database.update_package(package.entry) + self.database.commit() + + @under_lock + def upgrade(self, + expression: Optional[str] = None, + repotag: Optional[str] = None, + tarball: Optional[str] = None, + **kwargs): + """ Upgrade SONiC Package from either an expression representing + the package and its version, repository and tag or digest in + same format as "docker pulL" accepts or an image tarball path. + + Args: + expression: SONiC Package reference expression + repotag: Upgrade from REPO[:TAG][@DIGEST] + tarball: Upgrade from tarball, path to tarball file + kwargs: Upgrade options for self.upgrade_from_source + Raises: + PackageManagerError + """ + + source = self.get_package_source(expression, repotag, tarball) + self.upgrade_from_source(source, **kwargs) + + @under_lock + def upgrade_from_source(self, + source: PackageSource, + force=False, + skip_cli_plugin_installation=False): + """ Upgrade SONiC Package to a version the package reference + expression specifies. Can force the upgrade if force parameter + is True. Force can allow a package downgrade. + + Args: + source: SONiC Package source + force: Force the upgrade. + skip_cli_plugin_installation: Skip CLI plugin installation. + Raises: + PackageManagerError + """ + + new_package = source.get_package() + name = new_package.name + + with failure_ignore(force): + if not self.is_installed(name): + raise PackageUpgradeError(f'{name} is not installed') + + old_package = self.get_installed_package(name) + + if old_package.built_in: + raise PackageUpgradeError(f'Cannot upgrade built-in package {old_package.name}') + + old_feature = old_package.manifest['service']['name'] + new_feature = new_package.manifest['service']['name'] + old_version = old_package.manifest['package']['version'] + new_version = new_package.manifest['package']['version'] + + with failure_ignore(force): + if old_version == new_version: + raise PackageUpgradeError(f'{new_version} is already installed') + + # TODO: Not all packages might support downgrade. + # We put a check here but we understand that for some packages + # the downgrade might be safe to do. In that case we might want to + # add another argument to this function: allow_downgrade: bool = False. + # Another way to do that might be a variable in manifest describing package + # downgrade ability or downgrade-able versions. + if new_version < old_version: + raise PackageUpgradeError(f'Request to downgrade from {old_version} to {new_version}. ' + f'Downgrade might be not supported by the package') + + # remove currently installed package from the list + installed_packages = self._get_installed_packages_and(new_package) + + with failure_ignore(force): + validate_package_base_os_constraints(new_package, self.version_info) + validate_package_tree(installed_packages) + validate_package_cli_can_be_skipped(new_package, skip_cli_plugin_installation) + + # After all checks are passed we proceed to actual upgrade + + try: + with contextlib.ExitStack() as exit_stack: + self._uninstall_cli_plugins(old_package) + exit_stack.callback(rollback_wrapper(self._install_cli_plugins, old_package)) + + source.install(new_package) + exit_stack.callback(rollback_wrapper(source.uninstall, new_package)) + + if self.feature_registry.is_feature_enabled(old_feature): + self._systemctl_action(old_package, 'stop') + exit_stack.callback(rollback_wrapper(self._systemctl_action, + old_package, 'start')) + + self.service_creator.remove(old_package, deregister_feature=False) + exit_stack.callback(rollback_wrapper(self.service_creator.create, + old_package, register_feature=False)) + + # This is no return point, after we start removing old Docker images + # there is no guaranty we can actually successfully roll-back. + + # Clean containers based on the old image + containers = self.docker.ps(filters={'ancestor': old_package.image_id}, all=True) + for container in containers: + self.docker.rm(container.id, force=True) + + self.docker.rmi(old_package.image_id, force=True) + + self.service_creator.create(new_package, register_feature=False) + + if self.feature_registry.is_feature_enabled(new_feature): + self._systemctl_action(new_package, 'start') + + if not skip_cli_plugin_installation: + self._install_cli_plugins(new_package) + + exit_stack.pop_all() + except Exception as err: + raise PackageUpgradeError(f'Failed to upgrade {new_package.name}: {err}') + except KeyboardInterrupt: + raise + + new_package_entry = new_package.entry + new_package_entry.installed = True + new_package_entry.version = new_version + self.database.update_package(new_package_entry) + self.database.commit() + + @under_lock + def migrate_packages(self, + old_package_database: PackageDatabase, + dockerd_sock: Optional[str] = None): + """ Migrate packages from old database. This function can + do a comparison between current database and the database + passed in as argument. + If the package is missing in the current database it will be added. + If the package is installed in the passed database and in the current + it is not installed it will be installed with a passed database package version. + If the package is installed in the passed database and it is installed + in the current database but with older version the package will be upgraded to + the never version. + If the package is installed in the passed database and in the current + it is installed but with never version - no actions are taken. + If dockerd_sock parameter is passed, the migration process will use loaded + images from docker library of the currently installed image. + + Args: + old_package_database: SONiC Package Database to migrate packages from. + dockerd_sock: Path to dockerd socket. + Raises: + PackageManagerError + """ + + self._migrate_package_database(old_package_database) + + def migrate_package(old_package_entry, + new_package_entry, + migrate_operation=None): + """ Migrate package routine + + Args: + old_package_entry: Entry in old package database. + new_package_entry: Entry in new package database. + migrate_operation: Operation to perform: install or upgrade. + """ + + try: + migrate_func = { + 'install': self.install, + 'upgrade': self.upgrade, + }[migrate_operation] + except KeyError: + raise ValueError(f'Invalid operation passed in {migrate_operation}') + + name = new_package_entry.name + version = new_package_entry.version + + if dockerd_sock: + # dockerd_sock is defined, so use docked_sock to connect to + # dockerd and fetch package image from it. + log.info(f'{migrate_operation} {name} from old docker library') + docker_api = DockerApi(docker.DockerClient(base_url=f'unix://{dockerd_sock}')) + + image = docker_api.get_image(old_package_entry.image_id) + + with tempfile.NamedTemporaryFile('wb') as file: + for chunk in image.save(named=True): + file.write(chunk) + + migrate_func(tarball=file.name) + else: + log.info(f'{migrate_operation} {name} version {version}') + + migrate_func(f'{name}=={version}') + + # TODO: Topological sort packages by their dependencies first. + for old_package in old_package_database: + if not old_package.installed or old_package.built_in: + continue + + log.info(f'migrating package {old_package.name}') + + new_package = self.database.get_package(old_package.name) + + if new_package.installed: + if old_package.version > new_package.version: + log.info(f'{old_package.name} package version is greater ' + f'then installed in new image: ' + f'{old_package.version} > {new_package.version}') + log.info(f'upgrading {new_package.name} to {old_package.version}') + new_package.version = old_package.version + migrate_package(old_package, new_package, 'upgrade') + else: + log.info(f'skipping {new_package.name} as installed version is newer') + elif new_package.default_reference is not None: + new_package_ref = PackageReference(new_package.name, new_package.default_reference) + package_source = self.get_package_source(package_ref=new_package_ref) + package = package_source.get_package() + new_package_default_version = package.manifest['package']['version'] + if old_package.version > new_package_default_version: + log.info(f'{old_package.name} package version is lower ' + f'then the default in new image: ' + f'{old_package.version} > {new_package_default_version}') + new_package.version = old_package.version + migrate_package(old_package, new_package, 'install') + else: + self.install(f'{new_package.name}=={new_package_default_version}') + else: + # No default version and package is not installed. + # Migrate old package same version. + new_package.version = old_package.version + migrate_package(old_package, new_package, 'install') + + self.database.commit() + + def get_installed_package(self, name: str) -> Package: + """ Get installed package by name. + + Args: + name: package name. + Returns: + Package object. + """ + + package_entry = self.database.get_package(name) + source = LocalSource(package_entry, + self.database, + self.docker, + self.metadata_resolver) + return source.get_package() + + def get_package_source(self, + package_expression: Optional[str] = None, + repository_reference: Optional[str] = None, + tarboll_path: Optional[str] = None, + package_ref: Optional[PackageReference] = None): + """ Returns PackageSource object based on input source. + + Args: + package_expression: SONiC Package expression string + repository_reference: Install from REPO[:TAG][@DIGEST] + tarboll_path: Install from image tarball + package_ref: Package reference object + Returns: + SONiC Package object. + Raises: + ValueError if no source specified. + """ + + if package_expression: + ref = parse_reference_expression(package_expression) + return self.get_package_source(package_ref=ref) + elif repository_reference: + repo_ref = DockerReference.parse(repository_reference) + repository = repo_ref['name'] + reference = repo_ref['tag'] or repo_ref['digest'] + reference = reference or 'latest' + return RegistrySource(repository, + reference, + self.database, + self.docker, + self.metadata_resolver) + elif tarboll_path: + return TarballSource(tarboll_path, + self.database, + self.docker, + self.metadata_resolver) + elif package_ref: + package_entry = self.database.get_package(package_ref.name) + + # Determine the reference if not specified. + # If package is installed assume the installed + # one is requested, otherwise look for default + # reference defined for this package. In case package + # does not have a default reference raise an error. + if package_ref.reference is None: + if package_entry.installed: + return LocalSource(package_entry, + self.database, + self.docker, + self.metadata_resolver) + if package_entry.default_reference is not None: + package_ref.reference = package_entry.default_reference + else: + raise PackageManagerError(f'No default reference tag. ' + f'Please specify the version or tag explicitly') + + return RegistrySource(package_entry.repository, + package_ref.reference, + self.database, + self.docker, + self.metadata_resolver) + else: + raise ValueError('No package source provided') + + def get_package_available_versions(self, + name: str, + all: bool = False) -> Iterable: + """ Returns a list of available versions for package. + + Args: + name: Package name. + all: If set to True will return all tags including + those which do not follow semantic versioning. + Returns: + List of versions + """ + package_info = self.database.get_package(name) + registry = self.registry_resolver.get_registry_for(package_info.repository) + available_tags = registry.tags(package_info.repository) + + def is_semantic_ver_tag(tag: str) -> bool: + try: + tag_to_version(tag) + return True + except ValueError: + pass + return False + + if all: + return available_tags + + return map(tag_to_version, filter(is_semantic_ver_tag, available_tags)) + + def is_installed(self, name: str) -> bool: + """ Returns boolean whether a package called name is installed. + + Args: + name: Package name. + Returns: + True if package is installed, False otherwise. + """ + + if not self.database.has_package(name): + return False + package_info = self.database.get_package(name) + return package_info.installed + + def get_installed_packages(self) -> Dict[str, Package]: + """ Returns a dictionary of installed packages where + keys are package names and values are package objects. + + Returns: + Installed packages dictionary. + """ + + return { + entry.name: self.get_installed_package(entry.name) + for entry in self.database if entry.installed + } + + def _migrate_package_database(self, old_package_database: PackageDatabase): + """ Performs part of package migration process. + For every package in old_package_database that is not listed in current + database add a corresponding entry to current database. """ + + for package in old_package_database: + if not self.database.has_package(package.name): + self.database.add_package(package.name, + package.repository, + package.description, + package.default_reference) + + def _get_installed_packages_and(self, package: Package) -> Dict[str, Package]: + """ Returns a dictionary of installed packages with their names as keys + adding a package provided in the argument. """ + + packages = self.get_installed_packages() + packages[package.name] = package + return packages + + def _get_installed_packages_except(self, package: Package) -> Dict[str, Package]: + """ Returns a dictionary of installed packages with their names as keys + removing a package provided in the argument. """ + + packages = self.get_installed_packages() + packages.pop(package.name) + return packages + + # TODO: Replace with "config feature" command. + # The problem with current "config feature" command + # is that it is asynchronous, thus can't be used + # for package upgrade purposes where we need to wait + # till service stops before upgrading docker image. + # It would be really handy if we could just call + # something like: "config feature state --wait" + # instead of operating on systemd service since + # this is basically a duplicated code from "hostcfgd". + def _systemctl_action(self, package: Package, action: str): + """ Execute systemctl action for a service supporting + multi-asic services. """ + + name = package.manifest['service']['name'] + host_service = package.manifest['service']['host-service'] + asic_service = package.manifest['service']['asic-service'] + single_instance = host_service or (asic_service and not self.is_multi_npu) + multi_instance = asic_service and self.is_multi_npu + + if in_chroot(): + return + + if single_instance: + run_command(f'systemctl {action} {name}') + if multi_instance: + for npu in range(self.num_npus): + run_command(f'systemctl {action} {name}@{npu}') + + @staticmethod + def _get_cli_plugin_name(package: Package): + return utils.make_python_identifier(package.name) + '.py' + + @classmethod + def _get_cli_plugin_path(cls, package: Package, command): + pkg_loader = pkgutil.get_loader(f'{command}.plugins') + if pkg_loader is None: + raise PackageManagerError(f'Failed to get plugins path for {command} CLI') + plugins_pkg_path = os.path.dirname(pkg_loader.path) + return os.path.join(plugins_pkg_path, cls._get_cli_plugin_name(package)) + + def _install_cli_plugins(self, package: Package): + for command in ('show', 'config', 'clear'): + self._install_cli_plugin(package, command) + + def _uninstall_cli_plugins(self, package: Package): + for command in ('show', 'config', 'clear'): + self._uninstall_cli_plugin(package, command) + + def _install_cli_plugin(self, package: Package, command: str): + image_plugin_path = package.manifest['cli'][command] + if not image_plugin_path: + return + host_plugin_path = self._get_cli_plugin_path(package, command) + self.docker.extract(package.entry.image_id, image_plugin_path, host_plugin_path) + + def _uninstall_cli_plugin(self, package: Package, command: str): + image_plugin_path = package.manifest['cli'][command] + if not image_plugin_path: + return + host_plugin_path = self._get_cli_plugin_path(package, command) + if os.path.exists(host_plugin_path): + os.remove(host_plugin_path) + + @staticmethod + def get_manager() -> 'PackageManager': + """ Creates and returns PackageManager instance. + + Returns: + PackageManager + """ + + docker_api = DockerApi(docker.from_env()) + registry_resolver = RegistryResolver() + return PackageManager(DockerApi(docker.from_env(), ProgressManager()), + registry_resolver, + PackageDatabase.from_file(), + MetadataResolver(docker_api, registry_resolver), + ServiceCreator(FeatureRegistry(SonicDB), SonicDB), + device_info, + filelock.FileLock(PACKAGE_MANAGER_LOCK_FILE, timeout=0)) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py new file mode 100644 index 0000000000..74da06a956 --- /dev/null +++ b/sonic_package_manager/manifest.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python +from abc import ABC +from dataclasses import dataclass +from typing import Optional, List, Dict, Any + +from sonic_package_manager.constraint import ( + ComponentConstraints, + PackageConstraint, + VersionConstraint +) +from sonic_package_manager.errors import ManifestError +from sonic_package_manager.version import Version, VersionRange + + +class ManifestSchema: + """ ManifestSchema class describes and provides marshalling + and unmarshalling methods. + """ + + class Marshaller: + """ Base class for marshaling and un-marshaling. """ + + def marshal(self, value): + """ Validates and returns a valid manifest dictionary. + + Args: + value: input value to validate. + Returns: valid manifest node. + """ + + raise NotImplementedError + + def unmarshal(self, value): + """ Un-marshals the manifest to a dictionary. + + Args: + value: input value to validate. + Returns: valid manifest node. + """ + + raise NotImplementedError + + @dataclass + class ParsedMarshaller(Marshaller): + """ Marshaller used on types which support class method "parse" """ + + type: Any + + def marshal(self, value): + try: + return self.type.parse(value) + except ValueError as err: + raise ManifestError(f'Failed to marshal {value}: {err}') + + def unmarshal(self, value): + try: + return str(value) + except Exception as err: + raise ManifestError(f'Failed to unmarshal {value}: {err}') + + @dataclass + class DefaultMarshaller(Marshaller): + """ Default marshaller that validates if the given + value is instance of given type. """ + + type: type + + def marshal(self, value): + if not isinstance(value, self.type): + raise ManifestError(f'{value} is not of type {self.type.__name__}') + return value + + def unmarshal(self, value): + return value + + @dataclass + class ManifestNode(Marshaller, ABC): + """ + Base class for any manifest object. + + Attrs: + key: String representing the key for this object. + """ + + key: str + + @dataclass + class ManifestRoot(ManifestNode): + items: List + + def marshal(self, value: Optional[dict]): + result = {} + if value is None: + value = {} + + for item in self.items: + next_value = value.get(item.key) + result[item.key] = item.marshal(next_value) + return result + + def unmarshal(self, value): + return_value = {} + for item in self.items: + return_value[item.key] = item.unmarshal(value[item.key]) + return return_value + + @dataclass + class ManifestField(ManifestNode): + type: Any + default: Optional[Any] = None + + def marshal(self, value): + if value is None: + if self.default is not None: + return self.default + raise ManifestError(f'{self.key} is a required field but it is missing') + try: + return_value = self.type.marshal(value) + except Exception as err: + raise ManifestError(f'Failed to marshal {self.key}: {err}') + return return_value + + def unmarshal(self, value): + return self.type.unmarshal(value) + + @dataclass + class ManifestArray(ManifestNode): + type: Any + + def marshal(self, value): + if value is None: + return [] + + return_value = [] + try: + for item in value: + return_value.append(self.type.marshal(item)) + except Exception as err: + raise ManifestError(f'Failed to convert {self.key}={value} to array: {err}') + + return return_value + + def unmarshal(self, value): + return [self.type.unmarshal(item) for item in value] + + # TODO: add description for each field + SCHEMA = ManifestRoot('root', [ + ManifestField('version', ParsedMarshaller(Version), Version(1, 0, 0)), + ManifestRoot('package', [ + ManifestField('version', ParsedMarshaller(Version)), + ManifestField('name', DefaultMarshaller(str)), + ManifestField('description', DefaultMarshaller(str), ''), + ManifestField('base-os', ParsedMarshaller(ComponentConstraints), dict()), + ManifestArray('depends', ParsedMarshaller(PackageConstraint)), + ManifestArray('breaks', ParsedMarshaller(PackageConstraint)), + ManifestField('init-cfg', DefaultMarshaller(dict), dict()), + ManifestField('changelog', DefaultMarshaller(dict), dict()), + ManifestField('debug-dump', DefaultMarshaller(str), ''), + ]), + ManifestRoot('service', [ + ManifestField('name', DefaultMarshaller(str)), + ManifestArray('requires', DefaultMarshaller(str)), + ManifestArray('requisite', DefaultMarshaller(str)), + ManifestArray('wanted-by', DefaultMarshaller(str)), + ManifestArray('after', DefaultMarshaller(str)), + ManifestArray('before', DefaultMarshaller(str)), + ManifestArray('dependent', DefaultMarshaller(str)), + ManifestArray('dependent-of', DefaultMarshaller(str)), + ManifestField('post-start-action', DefaultMarshaller(str), ''), + ManifestField('pre-shutdown-action', DefaultMarshaller(str), ''), + ManifestField('asic-service', DefaultMarshaller(bool), False), + ManifestField('host-service', DefaultMarshaller(bool), True), + ManifestField('delayed', DefaultMarshaller(bool), False), + ]), + ManifestRoot('container', [ + ManifestField('privileged', DefaultMarshaller(bool), False), + ManifestArray('volumes', DefaultMarshaller(str)), + ManifestArray('mounts', ManifestRoot('mounts', [ + ManifestField('source', DefaultMarshaller(str)), + ManifestField('target', DefaultMarshaller(str)), + ManifestField('type', DefaultMarshaller(str)), + ])), + ManifestField('environment', DefaultMarshaller(dict), dict()), + ManifestArray('tmpfs', DefaultMarshaller(str)), + ]), + ManifestArray('processes', ManifestRoot('processes', [ + ManifestField('critical', DefaultMarshaller(bool)), + ManifestField('name', DefaultMarshaller(str)), + ManifestField('command', DefaultMarshaller(str)), + ])), + ManifestRoot('cli', [ + ManifestField('mandatory', DefaultMarshaller(bool), False), + ManifestField('show', DefaultMarshaller(str), ''), + ManifestField('config', DefaultMarshaller(str), ''), + ManifestField('clear', DefaultMarshaller(str), '') + ]) + ]) + + +class Manifest(dict): + """ Manifest object. """ + + SCHEMA = ManifestSchema.SCHEMA + + @classmethod + def marshal(cls, input_dict: dict): + return Manifest(cls.SCHEMA.marshal(input_dict)) + + def unmarshal(self) -> Dict: + return self.SCHEMA.unmarshal(self) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py new file mode 100644 index 0000000000..e5f9dbb3a5 --- /dev/null +++ b/sonic_package_manager/metadata.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python + +from dataclasses import dataclass, field + +import json +import tarfile +from typing import Dict + +from sonic_package_manager.errors import MetadataError +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.version import Version + + +def deep_update(dst: Dict, src: Dict) -> Dict: + """ Deep update dst dictionary with src dictionary. + + Args: + dst: Dictionary to update + src: Dictionary to update with + + Returns: + New merged dictionary. + """ + + for key, value in src.iteritems(): + if isinstance(value, dict): + node = dst.setdefault(key, {}) + deep_update(node, value) + else: + dst[key] = value + return dst + + +def translate_plain_to_tree(plain: Dict[str, str], sep='.') -> Dict: + """ Convert plain key/value dictionary into + a tree by spliting the key with '.' + + Args: + plain: Dictionary to convert into tree-like structure. + Keys in this dictionary have to be in a format: + "[key0].+", e.g: "com.azure.sonic" that + will be converted into tree like struct: + + { + "com": { + "azure": { + "sonic": {} + } + } + } + sep: Seperator string + + Returns: + Tree like structure + + """ + + res = {} + for key, value in plain.items(): + if sep not in key: + res[key] = value + continue + namespace, key = key.split(sep, 1) + res.setdefault(key, {}) + deep_update(res[key], translate_plain_to_tree({key: value})) + return res + + +@dataclass +class Metadata: + """ Package metadata object that can be retrieved from + OCI image manifest. """ + + manifest: Manifest + components: Dict[str, Version] = field(default_factory=dict) + + +class MetadataResolver: + """ Resolve metadata for package from different sources. """ + + def __init__(self, docker, registry_resolver): + self.docker = docker + self.registry_resolver = registry_resolver + + def from_local(self, image: str) -> Metadata: + """ Reads manifest from locally installed docker image. + + Args: + image: Docker image ID + Returns: + Metadata + Raises: + MetadataError + """ + + labels = self.docker.labels(image) + if labels is None: + raise MetadataError('No manifest found in image labels') + + return self.from_labels(labels) + + def from_registry(self, + repository: str, + reference: str) -> Metadata: + """ Reads manifest from remote registry. + + Args: + repository: Repository to pull image from + reference: Reference, either tag or digest + Returns: + Metadata + Raises: + MetadataError + """ + + registry = self.registry_resolver.get_registry_for(repository) + + manifest = registry.manifest(repository, reference) + digest = manifest['config']['digest'] + + blob = registry.blobs(repository, digest) + labels = blob['config']['Labels'] + if labels is None: + raise MetadataError('No manifest found in image labels') + + return self.from_labels(labels) + + def from_tarball(self, image_path: str) -> Metadata: + """ Reads manifest image tarball. + Args: + image_path: Path to image tarball. + Returns: + Manifest + Raises: + MetadataError + """ + + with tarfile.open(image_path) as image: + manifest = json.loads(image.extractfile('manifest.json').read()) + + blob = manifest[0]['Config'] + image_config = json.loads(image.extractfile(blob).read()) + labels = image_config['config']['Labels'] + if labels is None: + raise MetadataError('No manifest found in image labels') + + return self.from_labels(labels) + + @classmethod + def from_labels(cls, labels: Dict[str, str]) -> Metadata: + """ Get manifest from image labels. + + Args: + labels: key, value string pairs + Returns: + Metadata + Raises: + MetadataError + """ + + metadata_dict = translate_plain_to_tree(labels) + try: + sonic_metadata = metadata_dict['com']['azure']['sonic'] + except KeyError: + raise MetadataError('No metadata found in image labels') + + try: + manifest_string = sonic_metadata['manifest'] + except KeyError: + raise MetadataError('No manifest found in image labels') + + try: + manifest_dict = json.loads(manifest_string) + except (ValueError, TypeError) as err: + raise MetadataError(f'Failed to parse manifest JSON: {err}') + + components = {} + if 'versions' in sonic_metadata: + for component, version in sonic_metadata['versions'].items(): + try: + components[component] = Version.parse(version) + except ValueError as err: + raise MetadataError(f'Failed to parse component version: {err}') + + return Metadata(Manifest.marshal(manifest_dict), components) diff --git a/sonic_package_manager/package.py b/sonic_package_manager/package.py new file mode 100644 index 0000000000..2928f17392 --- /dev/null +++ b/sonic_package_manager/package.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +from dataclasses import dataclass + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.metadata import Metadata + + +@dataclass +class Package: + """ Package class is a representation of Package. + + Attributes: + entry: Package entry in package database + metadata: Metadata object for this package + manifest: Manifest for this package + components: Components versions for this package + name: Name of the package from package database + repository: Default repository to pull this package from + image_id: Docker image ID of the installed package; + It is set to None if package is not installed. + installed: Boolean flag whether package is installed or not. + build_in: Boolean flag whether package is built in or not. + + """ + + entry: PackageEntry + metadata: Metadata + + @property + def name(self): return self.entry.name + + @property + def repository(self): return self.entry.repository + + @property + def image_id(self): return self.entry.image_id + + @property + def installed(self): return self.entry.installed + + @property + def built_in(self): return self.entry.built_in + + @property + def version(self): return self.entry.version + + @property + def manifest(self): return self.metadata.manifest + + @property + def components(self): return self.metadata.components + diff --git a/sonic_package_manager/progress.py b/sonic_package_manager/progress.py new file mode 100644 index 0000000000..5258ebab98 --- /dev/null +++ b/sonic_package_manager/progress.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +import enlighten + +BAR_FMT = '{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}.2f}/{total:.2f}{unit_pad}{unit} ' + \ + '[{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' + +COUNTER_FMT = '{desc}{desc_pad}{count:.1f} {unit}{unit_pad}' + \ + '[{elapsed}, {rate:.2f}{unit_pad}{unit}/s]{fill}' + + +class ProgressManager: + """ ProgressManager is used for creating multiple progress bars + which nicely interact with logging and prints. """ + + def __init__(self): + self.manager = enlighten.get_manager() + self.pbars = {} + + def __enter__(self): + return self.manager.__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + return self.manager.__exit__(exc_type, exc_val, exc_tb) + + def new(self, id: str, *args, **kwargs): + """ Creates new progress bar with id. + Args: + id: progress bar identifier + *args: pass arguments for progress bar creation + **kwargs: pass keyword arguments for progress bar creation. + """ + + if 'bar_format' not in kwargs: + kwargs['bar_format'] = BAR_FMT + if 'counter_format' not in kwargs: + kwargs['counter_format'] = COUNTER_FMT + + self.pbars[id] = self.manager.counter(*args, **kwargs) + + def get(self, id: str): + """ Returns progress bar by id. + Args: + id: progress bar identifier + Returns: + Progress bar. + """ + + return self.pbars[id] + + def __contains__(self, id): + return id in self.pbars diff --git a/sonic_package_manager/reference.py b/sonic_package_manager/reference.py new file mode 100644 index 0000000000..9c4d8e825c --- /dev/null +++ b/sonic_package_manager/reference.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import re +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class PackageReference: + """ PackageReference is a package version constraint. """ + + name: str + reference: Optional[str] = None + + def __str__(self): + return f'{self.name} {self.reference}' + + @staticmethod + def parse(expression: str) -> 'PackageReference': + REQUIREMENT_SPECIFIER_RE = \ + r'(?P[A-Za-z0-9_-]+)(?P@(?P.*))' + + match = re.match(REQUIREMENT_SPECIFIER_RE, expression) + if match is None: + raise ValueError(f'Invalid reference specifier {expression}') + groupdict = match.groupdict() + name = groupdict.get('name') + reference = groupdict.get('reference') + + return PackageReference(name, reference) diff --git a/sonic_package_manager/registry.py b/sonic_package_manager/registry.py new file mode 100644 index 0000000000..bf4308efa0 --- /dev/null +++ b/sonic_package_manager/registry.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +import json +from dataclasses import dataclass +from typing import List, Dict + +import requests +import www_authenticate +from docker_image import reference +from prettyprinter import pformat + +from sonic_package_manager.logger import log +from sonic_package_manager.utils import DockerReference + + +class AuthenticationServiceError(Exception): + """ Exception class for errors related to authentication. """ + + pass + + +class AuthenticationService: + """ AuthenticationService provides an authentication tokens. """ + + @staticmethod + def get_token(realm, service, scope) -> str: + """ Retrieve an authentication token. + + Args: + realm: Realm: url to request token. + service: service to request token for. + scope: scope to requests token for. + Returns: + token value as a string. + """ + + log.debug(f'getting authentication token: realm={realm} service={service} scope={scope}') + + response = requests.get(f'{realm}?scope={scope}&service={service}') + if response.status_code != requests.codes.ok: + raise AuthenticationServiceError(f'Failed to retrieve token') + + content = json.loads(response.content) + token = content['token'] + expires_in = content['expires_in'] + + log.debug(f'authentication token for realm={realm} service={service} scope={scope}: ' + f'token={token} expires_in={expires_in}') + + return token + + +@dataclass +class RegistryApiError(Exception): + """ Class for registry related errors. """ + + msg: str + response: requests.Response + + def __str__(self): + code = self.response.status_code + content = self.response.content.decode() + try: + content = json.loads(content) + except ValueError: + pass + return f'{self.msg}: code: {code} details: {pformat(content)}' + + +class Registry: + """ Provides a Docker registry interface. """ + + MIME_DOCKER_MANIFEST = 'application/vnd.docker.distribution.manifest.v2+json' + + def __init__(self, host: str): + self.url = host + + @staticmethod + def _execute_get_request(url, headers): + response = requests.get(url, headers=headers) + if response.status_code == requests.codes.unauthorized: + # Get authentication details from headers + # Registry should tell how to authenticate + www_authenticate_details = response.headers['Www-Authenticate'] + log.debug(f'unauthorized: retrieving authentication details ' + f'from response headers {www_authenticate_details}') + bearer = www_authenticate.parse(www_authenticate_details)['bearer'] + token = AuthenticationService.get_token(**bearer) + headers['Authorization'] = f'Bearer {token}' + # Repeat request + response = requests.get(url, headers=headers) + return response + + def _get_base_url(self, repository: str): + return f'{self.url}/v2/{repository}' + + def tags(self, repository: str) -> List[str]: + log.debug(f'getting tags for {repository}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = {'Accept': 'application/json'} + url = f'{self._get_base_url(repository)}/tags/list' + response = self._execute_get_request(url, headers) + if response.status_code != requests.codes.ok: + raise RegistryApiError(f'Failed to retrieve tags from {repository}', response) + + content = json.loads(response.content) + log.debug(f'tags list api response: f{content}') + + return content['tags'] + + def manifest(self, repository: str, ref: str) -> Dict: + log.debug(f'getting manifest for {repository}:{ref}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = {'Accept': self.MIME_DOCKER_MANIFEST} + url = f'{self._get_base_url(repository)}/manifests/{ref}' + response = self._execute_get_request(url, headers) + + if response.status_code != requests.codes.ok: + raise RegistryApiError(f'Failed to retrieve manifest for {repository}:{ref}', response) + + content = json.loads(response.content) + log.debug(f'manifest content for {repository}:{ref}: {content}') + + return content + + def blobs(self, repository: str, digest: str): + log.debug(f'retrieving blob for {repository}:{digest}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = {'Accept': self.MIME_DOCKER_MANIFEST} + url = f'{self._get_base_url(repository)}/blobs/{digest}' + response = self._execute_get_request(url, headers) + if response.status_code != requests.codes.ok: + raise RegistryApiError(f'Failed to retrieve blobs for {repository}:{digest}', response) + content = json.loads(response.content) + + log.debug(f'retrieved blob for {repository}:{digest}: {content}') + return content + + +class RegistryResolver: + """ Returns a registry object based on the input repository reference + string. """ + + DockerHubRegistry = Registry('https://index.docker.io') + + def __init__(self): + pass + + def get_registry_for(self, ref: str) -> Registry: + domain, _ = DockerReference.split_docker_domain(ref) + if domain == reference.DEFAULT_DOMAIN: + return self.DockerHubRegistry + # TODO: support insecure registries + return Registry(f'https://{domain}') diff --git a/sonic_package_manager/service_creator/__init__.py b/sonic_package_manager/service_creator/__init__.py new file mode 100644 index 0000000000..e2af81ceb5 --- /dev/null +++ b/sonic_package_manager/service_creator/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python + +ETC_SONIC_PATH = '/etc/sonic' diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py new file mode 100644 index 0000000000..f62d0a3074 --- /dev/null +++ b/sonic_package_manager/service_creator/creator.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python +import contextlib +import os +import stat +import subprocess +from typing import Dict + +import jinja2 as jinja2 +from prettyprinter import pformat + +from sonic_package_manager.logger import log +from sonic_package_manager.package import Package +from sonic_package_manager.service_creator import ETC_SONIC_PATH +from sonic_package_manager.service_creator.feature import FeatureRegistry +from sonic_package_manager.service_creator.utils import in_chroot + +SERVICE_FILE_TEMPLATE = 'sonic.service.j2' +TIMER_UNIT_TEMPLATE = 'timer.unit.j2' + +SYSTEMD_LOCATION = '/usr/lib/systemd/system' + +SERVICE_MGMT_SCRIPT_TEMPLATE = 'service_mgmt.sh.j2' +SERVICE_MGMT_SCRIPT_LOCATION = '/usr/local/bin' + +DOCKER_CTL_SCRIPT_TEMPLATE = 'docker_image_ctl.j2' +DOCKER_CTL_SCRIPT_LOCATION = '/usr/bin' + +MONIT_CONF_TEMPLATE = 'monit.conf.j2' +MONIT_CONF_LOCATION = '/etc/monit/conf.d/' + +DEBUG_DUMP_SCRIPT_TEMPLATE = 'dump.sh.j2' +DEBUG_DUMP_SCRIPT_LOCATION = '/usr/local/bin/debug-dump/' + +TEMPLATES_PATH = '/usr/share/sonic/templates' + + +class ServiceCreatorError(Exception): + pass + + +def render_template(in_template: str, + outfile: str, + render_ctx: Dict, + executable: bool = False): + """ Template renderer helper routine. + Args: + in_template: Input file with template content + outfile: Output file to render template to + render_ctx: Dictionary used to generate jinja2 template + executable: Set executable bit on rendered file + """ + + log.debug(f'Rendering {in_template} to {outfile} with {pformat(render_ctx)}') + + with open(in_template, 'r') as instream: + template = jinja2.Template(instream.read()) + + with open(outfile, 'w') as outstream: + outstream.write(template.render(**render_ctx)) + + if executable: + set_executable_bit(outfile) + + +def get_tmpl_path(template_name: str) -> str: + """ Returns a path to a template. + Args: + template_name: Template file name. + """ + + return os.path.join(TEMPLATES_PATH, template_name) + + +def set_executable_bit(filepath): + """ Sets +x on filepath. """ + + st = os.stat(filepath) + os.chmod(filepath, st.st_mode | stat.S_IEXEC) + + +def run_command(command: str): + """ Run arbitrary bash command. + Args: + command: String command to execute as bash script + Raises: + PackageManagerError: Raised when the command return code + is not 0. + """ + + log.debug(f'running command: {command}') + + proc = subprocess.Popen(command, + shell=True, + executable='/bin/bash', + stdout=subprocess.PIPE) + (out, _) = proc.communicate() + if proc.returncode != 0: + raise ServiceCreatorError(f'Failed to execute "{command}"') + + +class ServiceCreator: + """ Creates and registers services in SONiC based on the package + manifest. """ + + def __init__(self, feature_registry: FeatureRegistry, sonic_db): + self.feature_registry = feature_registry + self.sonic_db = sonic_db + + def create(self, + package: Package, + register_feature=True, + state='enabled', + owner='local'): + try: + self.generate_container_mgmt(package) + self.generate_service_mgmt(package) + self.update_dependent_list_file(package) + self.generate_systemd_service(package) + self.generate_monit_conf(package) + self.generate_dump_script(package) + + self.set_initial_config(package) + + self.post_operation_hook() + + if register_feature: + self.feature_registry.register(package.manifest, + state, owner) + except (Exception, KeyboardInterrupt): + self.remove(package, not register_feature) + raise + + def remove(self, package: Package, deregister_feature=True): + name = package.manifest['service']['name'] + + def remove_file(path): + if os.path.exists(path): + os.remove(path) + log.info(f'removed {path}') + + remove_file(os.path.join(MONIT_CONF_LOCATION, f'monit_{name}')) + remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}.service')) + remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}@.service')) + remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) + remove_file(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) + remove_file(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) + + self.update_dependent_list_file(package, remove=True) + + self.post_operation_hook() + + if deregister_feature: + self.feature_registry.deregister(package.manifest['service']['name']) + + def post_operation_hook(self): + if not in_chroot(): + run_command('systemctl daemon-reload') + run_command('systemctl reload monit') + + def generate_container_mgmt(self, package: Package): + image_id = package.image_id + name = package.manifest['service']['name'] + container_spec = package.manifest['container'] + script_path = os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh') + script_template = get_tmpl_path(DOCKER_CTL_SCRIPT_TEMPLATE) + run_opt = [] + + if container_spec['privileged']: + run_opt.append('--privileged') + + run_opt.append('-t') + + for volume in container_spec['volumes']: + run_opt.append(f'-v {volume}') + + for mount in container_spec['mounts']: + mount_type, source, target = mount['type'], mount['source'], mount['target'] + run_opt.append(f'--mount type={mount_type},source={source},target={target}') + + for tmpfs_mount in container_spec['tmpfs']: + run_opt.append(f'--tmpfs {tmpfs_mount}') + + for env_name, value in container_spec['environment'].items(): + run_opt.append(f'-e {env_name}={value}') + + run_opt = ' '.join(run_opt) + render_ctx = { + 'docker_container_name': name, + 'docker_image_id': image_id, + 'docker_image_run_opt': run_opt, + } + render_template(script_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') + + def generate_service_mgmt(self, package: Package): + name = package.manifest['service']['name'] + multi_instance_services = self.feature_registry.get_multi_instance_features() + script_path = os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh') + scrip_template = get_tmpl_path(SERVICE_MGMT_SCRIPT_TEMPLATE) + render_ctx = { + 'source': get_tmpl_path(SERVICE_MGMT_SCRIPT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance_services': multi_instance_services, + } + render_template(scrip_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') + + def generate_systemd_service(self, package: Package): + name = package.manifest['service']['name'] + multi_instance_services = self.feature_registry.get_multi_instance_features() + + template = get_tmpl_path(SERVICE_FILE_TEMPLATE) + template_vars = { + 'source': get_tmpl_path(SERVICE_FILE_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance': False, + 'multi_instance_services': multi_instance_services, + } + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}.service') + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['asic-service']: + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}@.service') + template_vars['multi_instance'] = True + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['delayed']: + template_vars = { + 'source': get_tmpl_path(TIMER_UNIT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance': False, + } + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}.timer') + template = os.path.join(TEMPLATES_PATH, TIMER_UNIT_TEMPLATE) + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['asic-service']: + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}@.timer') + template_vars['multi_instance'] = True + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + def generate_monit_conf(self, package: Package): + name = package.manifest['service']['name'] + processes = package.manifest['processes'] + output_filename = os.path.join(MONIT_CONF_LOCATION, f'monit_{name}') + render_template(get_tmpl_path(MONIT_CONF_TEMPLATE), output_filename, + {'source': get_tmpl_path(MONIT_CONF_TEMPLATE), + 'feature': name, + 'processes': processes}) + log.info(f'generated {output_filename}') + + def update_dependent_list_file(self, package: Package, remove=False): + name = package.manifest['service']['name'] + dependent_of = package.manifest['service']['dependent-of'] + host_service = package.manifest['service']['host-service'] + asic_service = package.manifest['service']['asic-service'] + + def update_dependent(service, name, multi_inst): + if multi_inst: + filename = f'{service}_multi_inst_dependent' + else: + filename = f'{service}_dependent' + + filepath = os.path.join(ETC_SONIC_PATH, filename) + + dependent_services = set() + if os.path.exists(filepath): + with open(filepath) as fp: + dependent_services.update({line.strip() for line in fp.readlines()}) + if remove: + with contextlib.suppress(KeyError): + dependent_services.remove(name) + else: + dependent_services.add(name) + with open(filepath, 'w') as fp: + fp.write('\n'.join(dependent_services)) + + for service in dependent_of: + if host_service: + update_dependent(service, name, multi_inst=False) + if asic_service: + update_dependent(service, name, multi_inst=True) + + def generate_dump_script(self, package): + name = package.manifest['service']['name'] + + if not package.manifest['package']['debug-dump']: + return + + if not os.path.exists(DEBUG_DUMP_SCRIPT_LOCATION): + os.mkdir(DEBUG_DUMP_SCRIPT_LOCATION) + + scrip_template = os.path.join(TEMPLATES_PATH, DEBUG_DUMP_SCRIPT_TEMPLATE) + script_path = os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}') + render_ctx = { + 'source': get_tmpl_path(SERVICE_MGMT_SCRIPT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + } + render_template(scrip_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') + + def set_initial_config(self, package): + init_cfg = package.manifest['package']['init-cfg'] + + def get_tables(table_name): + tables = [] + + running_table = self.sonic_db.running_table(table_name) + if running_table is not None: + tables.append(running_table) + + persistent_table = self.sonic_db.persistent_table(table_name) + if persistent_table is not None: + tables.append(persistent_table) + + initial_table = self.sonic_db.initial_table(table_name) + if initial_table is not None: + tables.append(initial_table) + + return tables + + for tablename, content in init_cfg.items(): + if not isinstance(content, dict): + continue + + tables = get_tables(tablename) + + for key in content: + for table in tables: + cfg = content[key] + exists, old_fvs = table.get(key) + if exists: + cfg.update(old_fvs) + fvs = list(cfg.items()) + table.set(key, fvs) diff --git a/sonic_package_manager/service_creator/feature.py b/sonic_package_manager/service_creator/feature.py new file mode 100644 index 0000000000..4df06384d2 --- /dev/null +++ b/sonic_package_manager/service_creator/feature.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python + +""" This module implements new feature registration/de-registration in SONiC system. """ + +from typing import Dict, Type + +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.service_creator.sonic_db import SonicDB + +FEATURE = 'FEATURE' +DEFAULT_FEATURE_CONFIG = { + 'state': 'disabled', + 'auto_restart': 'enabled', + 'high_mem_alert': 'disabled', + 'set_owner': 'local' +} + + +class FeatureRegistry: + """ FeatureRegistry class provides an interface to + register/de-register new feature persistently. """ + + def __init__(self, sonic_db: Type[SonicDB]): + self._sonic_db = sonic_db + + def register(self, + manifest: Manifest, + state: str = 'disabled', + owner: str = 'local'): + name = manifest['service']['name'] + for table in self._get_tables(): + cfg_entries = self.get_default_feature_entries(state, owner) + non_cfg_entries = self.get_non_configurable_feature_entries(manifest) + + exists, current_cfg = table.get(name) + + new_cfg = cfg_entries.copy() + # Override configurable entries with CONFIG DB data. + new_cfg = {**new_cfg, **dict(current_cfg)} + # Override CONFIG DB data with non configurable entries. + new_cfg = {**new_cfg, **non_cfg_entries} + + table.set(name, list(new_cfg.items())) + + def deregister(self, name: str): + for table in self._get_tables(): + table._del(name) + + def is_feature_enabled(self, name: str) -> bool: + """ Returns whether the feature is current enabled + or not. Accesses running CONFIG DB. If no running CONFIG_DB + table is found in tables returns False. """ + + running_db_table = self._sonic_db.running_table(FEATURE) + if running_db_table is None: + return False + + exists, cfg = running_db_table.get(name) + if not exists: + return False + cfg = dict(cfg) + return cfg.get('state').lower() == 'enabled' + + def get_multi_instance_features(self): + res = [] + init_db_table = self._sonic_db.initial_table(FEATURE) + for feature in init_db_table.keys(): + exists, cfg = init_db_table.get(feature) + assert exists + cfg = dict(cfg) + asic_flag = str(cfg.get('has_per_asic_scope', 'False')) + if asic_flag.lower() == 'true': + res.append(feature) + return res + + @staticmethod + def get_default_feature_entries(state=None, owner=None) -> Dict[str, str]: + """ Get configurable feature table entries: + e.g. 'state', 'auto_restart', etc. """ + + cfg = DEFAULT_FEATURE_CONFIG.copy() + if state: + cfg['state'] = state + if owner: + cfg['set_owner'] = owner + return cfg + + @staticmethod + def get_non_configurable_feature_entries(manifest) -> Dict[str, str]: + """ Get non-configurable feature table entries: e.g. 'has_timer' """ + + return { + 'has_per_asic_scope': str(manifest['service']['asic-service']), + 'has_global_scope': str(manifest['service']['host-service']), + 'has_timer': str(manifest['service']['delayed']), + } + + def _get_tables(self): + tables = [] + running = self._sonic_db.running_table(FEATURE) + if running is not None: # it's Ok if there is no database container running + tables.append(running) + persistent = self._sonic_db.persistent_table(FEATURE) + if persistent is not None: # it's Ok if there is no config_db.json + tables.append(persistent) + tables.append(self._sonic_db.initial_table(FEATURE)) # init_cfg.json is must + + return tables diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py new file mode 100644 index 0000000000..a9ba837ab1 --- /dev/null +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +import contextlib +import json +import os + +from swsscommon import swsscommon + +from sonic_package_manager.service_creator import ETC_SONIC_PATH +from sonic_package_manager.service_creator.utils import in_chroot + +CONFIG_DB = 'CONFIG_DB' +CONFIG_DB_JSON = os.path.join(ETC_SONIC_PATH, 'config_db.json') +INIT_CFG_JSON = os.path.join(ETC_SONIC_PATH, 'init_cfg.json') + + +class FileDbTable: + """ swsscommon.Table adapter for persistent DBs. """ + + def __init__(self, file, table): + self._file = file + self._table = table + + def keys(self): + with open(self._file) as stream: + config = json.load(stream) + return config.get(self._table, {}).keys() + + def get(self, key): + with open(self._file) as stream: + config = json.load(stream) + + table = config.get(self._table, {}) + exists = key in table + fvs_dict = table.get(key, {}) + fvs = list(fvs_dict.items()) + return exists, fvs + + def set(self, key, fvs): + with open(self._file) as stream: + config = json.load(stream) + + table = config.setdefault(self._table, {}) + table.update({key: dict(fvs)}) + + with open(self._file, 'w') as stream: + json.dump(config, stream, indent=4) + + def _del(self, key): + with open(self._file) as stream: + config = json.load(stream) + + with contextlib.suppress(KeyError): + config[self._table].pop(key) + + with open(self._file, 'w') as stream: + json.dump(config, stream, indent=4) + + +class SonicDB: + """ Store different DB access objects for + running DB and also for persistent and initial + configs. """ + + _running = None + + @classmethod + def running_table(cls, table): + """ Returns running DB table. """ + + # In chroot we can connect to a running + # DB via TCP socket, we should ignore this case. + if in_chroot(): + return None + + if cls._running is None: + try: + cls._running = swsscommon.DBConnector(CONFIG_DB, 0) + except RuntimeError: + # Failed to connect to DB. + return None + + return swsscommon.Table(cls._running, table) + + @classmethod + def persistent_table(cls, table): + """ Returns persistent DB table. """ + + if not os.path.exists(CONFIG_DB_JSON): + return None + + return FileDbTable(CONFIG_DB_JSON, table) + + @classmethod + def initial_table(cls, table): + """ Returns initial DB table. """ + + return FileDbTable(INIT_CFG_JSON, table) diff --git a/sonic_package_manager/service_creator/utils.py b/sonic_package_manager/service_creator/utils.py new file mode 100644 index 0000000000..cdeeb17abb --- /dev/null +++ b/sonic_package_manager/service_creator/utils.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +import os + + +def in_chroot() -> bool: + """ Verify if we are running in chroot or not + by comparing root / mount point device id and inode + with init process - /proc/1/root mount point device + id and inode. If those match we are not chroot-ed + otherwise we are. """ + + root_stat = os.stat('/') + init_root_stat = os.stat('/proc/1/root') + + return (root_stat.st_dev, root_stat.st_ino) != \ + (init_root_stat.st_dev, init_root_stat.st_ino) diff --git a/sonic_package_manager/source.py b/sonic_package_manager/source.py new file mode 100644 index 0000000000..4720b5be9e --- /dev/null +++ b/sonic_package_manager/source.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 + +from sonic_package_manager.database import PackageDatabase, PackageEntry +from sonic_package_manager.dockerapi import DockerApi, get_repository_from_image +from sonic_package_manager.metadata import Metadata, MetadataResolver +from sonic_package_manager.package import Package + + +class PackageSource(object): + """ PackageSource abstracts the way manifest is read + and image is retrieved based on different image sources. + (i.e from registry, from tarball or locally installed) """ + + def __init__(self, + database: PackageDatabase, + docker: DockerApi, + metadata_resolver: MetadataResolver): + self.database = database + self.docker = docker + self.metadata_resolver = metadata_resolver + + def get_metadata(self) -> Metadata: + """ Returns package manifest. + Child class has to implement this method. + + Returns: + Metadata + """ + raise NotImplementedError + + def install_image(self): + """ Install image based on package source. + Child class has to implement this method. + + Returns: + Docker Image object. + """ + + raise NotImplementedError + + def install(self, package: Package): + """ Install image based on package source, + record installation infromation in PackageEntry.. + + Args: + package: SONiC Package + """ + + image = self.install_image() + package.entry.image_id = image.id + # if no repository is defined for this package + # get repository from image + if not package.repository: + package.entry.repository = get_repository_from_image(image) + + def uninstall(self, package: Package): + """ Uninstall image. + + Args: + package: SONiC Package + """ + + self.docker.rmi(package.image_id) + package.entry.image_id = None + + def get_package(self) -> Package: + """ Returns SONiC Package based on manifest. + + Returns: + SONiC Package + """ + + metadata = self.get_metadata() + manifest = metadata.manifest + + name = manifest['package']['name'] + description = manifest['package']['description'] + + repository = None + + if self.database.has_package(name): + # inherit package database info + package = self.database.get_package(name) + repository = package.repository + description = description or package.description + + return Package( + PackageEntry( + name, + repository, + description, + ), + metadata + ) + + +class TarballSource(PackageSource): + """ TarballSource implements PackageSource + for locally existing image saved as tarball. """ + + def __init__(self, + tarball_path: str, + database: PackageDatabase, + docker: DockerApi, + metadata_resolver: MetadataResolver): + super().__init__(database, + docker, + metadata_resolver) + self.tarball_path = tarball_path + + def get_metadata(self) -> Metadata: + """ Returns manifest read from tarball. """ + + return self.metadata_resolver.from_tarball(self.tarball_path) + + def install_image(self): + """ Installs image from local tarball source. """ + + return self.docker.load(self.tarball_path) + + +class RegistrySource(PackageSource): + """ RegistrySource implements PackageSource + for packages that are pulled from registry. """ + + def __init__(self, + repository: str, + reference: str, + database: PackageDatabase, + docker: DockerApi, + metadata_resolver: MetadataResolver): + super().__init__(database, + docker, + metadata_resolver) + self.repository = repository + self.reference = reference + + def get_metadata(self) -> Metadata: + """ Returns manifest read from registry. """ + + return self.metadata_resolver.from_registry(self.repository, + self.reference) + + def install_image(self): + """ Installs image from registry. """ + + return self.docker.pull(self.repository, self.reference) + + +class LocalSource(PackageSource): + """ LocalSource accesses local docker library to retrieve manifest + but does not implement installation of the image. """ + + def __init__(self, + entry: PackageEntry, + database: PackageDatabase, + docker: DockerApi, + metadata_resolver: MetadataResolver): + super().__init__(database, + docker, + metadata_resolver) + self.entry = entry + + def get_metadata(self) -> Metadata: + """ Returns manifest read from locally installed Docker. """ + + image = self.entry.image_id + + if self.entry.built_in: + # Built-in (installed not via sonic-package-manager) + # won't have image_id in database. Using their + # repository name as image. + image = f'{self.entry.repository}:latest' + + return self.metadata_resolver.from_local(image) + + def get_package(self) -> Package: + return Package(self.entry, self.get_metadata()) diff --git a/sonic_package_manager/utils.py b/sonic_package_manager/utils.py new file mode 100644 index 0000000000..410947dd24 --- /dev/null +++ b/sonic_package_manager/utils.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import keyword +import re + +from docker_image.reference import Reference + +DockerReference = Reference + + +def make_python_identifier(string): + """ + Takes an arbitrary string and creates a valid Python identifier. + + Identifiers must follow the convention outlined here: + https://docs.python.org/2/reference/lexical_analysis.html#identifiers + """ + + # create a working copy (and make it lowercase, while we're at it) + s = string.lower() + + # remove leading and trailing whitespace + s = s.strip() + + # Make spaces into underscores + s = re.sub('[\\s\\t\\n]+', '_', s) + + # Remove invalid characters + s = re.sub('[^0-9a-zA-Z_]', '', s) + + # Remove leading characters until we find a letter or underscore + s = re.sub('^[^a-zA-Z_]+', '', s) + + # Check that the string is not a python identifier + while s in keyword.kwlist: + if re.match(".*?_\d+$", s): + i = re.match(".*?_(\d+)$", s).groups()[0] + s = s.strip('_'+i) + '_'+str(int(i)+1) + else: + s += '_1' + + return s diff --git a/sonic_package_manager/version.py b/sonic_package_manager/version.py new file mode 100644 index 0000000000..e5a5623d3b --- /dev/null +++ b/sonic_package_manager/version.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +""" Version and helpers routines. """ + +import semver + +Version = semver.Version +VersionRange = semver.VersionRange + + +def version_to_tag(ver: Version) -> str: + """ Converts the version to Docker compliant tag string. """ + + return str(ver).replace('+', '_') + + +def tag_to_version(tag: str) -> Version: + """ Converts the version to Docker compliant tag string. """ + + try: + return Version.parse(tag.replace('_', '+')) + except ValueError as err: + raise ValueError(f'Failed to convert {tag} to version string: {err}') diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py new file mode 100644 index 0000000000..bf787bdab4 --- /dev/null +++ b/tests/sonic_package_manager/conftest.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python + +from dataclasses import dataclass +from unittest import mock +from unittest.mock import Mock, MagicMock + +import pytest +from docker_image.reference import Reference + +from sonic_package_manager.database import PackageDatabase, PackageEntry +from sonic_package_manager.manager import DockerApi, PackageManager +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.metadata import Metadata, MetadataResolver +from sonic_package_manager.registry import RegistryResolver +from sonic_package_manager.version import Version +from sonic_package_manager.service_creator.creator import * + + +@pytest.fixture +def mock_docker_api(): + docker = MagicMock(DockerApi) + + @dataclass + class Image: + id: str + + @property + def attrs(self): + return {'RepoTags': [self.id]} + + def pull(repo, ref): + return Image(f'{repo}:latest') + + def load(filename): + return Image(filename) + + docker.pull = MagicMock(side_effect=pull) + docker.load = MagicMock(side_effect=load) + + yield docker + + +@pytest.fixture +def mock_registry_resolver(): + yield Mock(RegistryResolver) + + +@pytest.fixture +def mock_metadata_resolver(): + yield Mock(MetadataResolver) + + +@pytest.fixture +def mock_feature_registry(): + yield MagicMock() + + +@pytest.fixture +def mock_service_creator(): + yield Mock() + + +@pytest.fixture +def mock_sonic_db(): + yield Mock() + + +@pytest.fixture +def fake_metadata_resolver(): + class FakeMetadataResolver: + def __init__(self): + self.metadata_store = {} + self.add('docker-database', 'latest', 'database', '1.0.0') + self.add('docker-orchagent', 'latest', 'swss', '1.0.0', + components={ + 'libswsscommon': Version.parse('1.0.0'), + 'libsairedis': Version.parse('1.0.0') + } + ) + self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') + self.add('Azure/docker-test-2', '1.5.0', 'test-package-2', '1.5.0') + self.add('Azure/docker-test-2', '2.0.0', 'test-package-2', '2.0.0') + self.add('Azure/docker-test-3', 'latest', 'test-package-3', '1.6.0') + self.add('Azure/docker-test-3', '1.5.0', 'test-package-3', '1.5.0') + self.add('Azure/docker-test-3', '1.6.0', 'test-package-3', '1.6.0') + self.add('Azure/docker-test-4', '1.5.0', 'test-package-4', '1.5.0') + self.add('Azure/docker-test-5', '1.5.0', 'test-package-5', '1.5.0') + self.add('Azure/docker-test-5', '1.9.0', 'test-package-5', '1.9.0') + self.add('Azure/docker-test-6', '1.5.0', 'test-package-6', '1.5.0') + self.add('Azure/docker-test-6', '1.9.0', 'test-package-6', '1.9.0') + self.add('Azure/docker-test-6', '2.0.0', 'test-package-6', '2.0.0') + self.add('Azure/docker-test-6', 'latest', 'test-package-6', '1.5.0') + + def from_registry(self, repository: str, reference: str): + manifest = Manifest.marshal(self.metadata_store[repository][reference]['manifest']) + components = self.metadata_store[repository][reference]['components'] + return Metadata(manifest, components) + + def from_local(self, image: str): + ref = Reference.parse(image) + manifest = Manifest.marshal(self.metadata_store[ref['name']][ref['tag']]['manifest']) + components = self.metadata_store[ref['name']][ref['tag']]['components'] + return Metadata(manifest, components) + + def from_tarball(self, filepath: str) -> Manifest: + path, ref = filepath.split(':') + manifest = Manifest.marshal(self.metadata_store[path][ref]['manifest']) + components = self.metadata_store[path][ref]['components'] + return Metadata(manifest, components) + + def add(self, repo, reference, name, version, components=None): + repo_dict = self.metadata_store.setdefault(repo, {}) + repo_dict[reference] = { + 'manifest': { + 'package': { + 'version': version, + 'name': name, + 'base-os': {}, + }, + 'service': { + 'name': name, + } + }, + 'components': components or {}, + } + + yield FakeMetadataResolver() + + +@pytest.fixture +def fake_device_info(): + class FakeDeviceInfo: + def __init__(self): + self.multi_npu = True + self.num_npus = 1 + self.version_info = { + 'libswsscommon': '1.0.0', + } + + def is_multi_npu(self): + return self.multi_npu + + def get_num_npus(self): + return self.num_npus + + def get_sonic_version_info(self): + return self.version_info + + yield FakeDeviceInfo() + + +def add_package(content, metadata_resolver, repository, reference, **kwargs): + metadata = metadata_resolver.from_registry(repository, reference) + name = metadata.manifest['package']['name'] + version = metadata.manifest['package']['version'] + installed = kwargs.get('installed', False) + built_in = kwargs.get('built-in', False) + + if installed and not built_in and 'image_id' not in kwargs: + kwargs['image_id'] = f'{repository}:{reference}' + + if installed and 'version' not in kwargs: + kwargs['version'] = version + + content[name] = PackageEntry(name, repository, **kwargs) + + +@pytest.fixture +def fake_db(fake_metadata_resolver): + content = {} + + add_package( + content, + fake_metadata_resolver, + 'docker-database', + 'latest', + description='SONiC database service', + default_reference='1.0.0', + installed=True, + built_in=True + ) + add_package( + content, + fake_metadata_resolver, + 'docker-orchagent', + 'latest', + description='SONiC switch state service', + default_reference='1.0.0', + installed=True, + built_in=True + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test', + '1.6.0', + description='SONiC Package Manager Test Package', + default_reference='1.6.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-2', + '1.5.0', + description='SONiC Package Manager Test Package #2', + default_reference='1.5.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-3', + '1.5.0', + description='SONiC Package Manager Test Package #3', + default_reference='1.5.0', + installed=True, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-5', + '1.9.0', + description='SONiC Package Manager Test Package #5', + default_reference='1.9.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-6', + '1.5.0', + description='SONiC Package Manager Test Package #6', + default_reference='1.5.0', + installed=False, + built_in=False + ) + + yield PackageDatabase(content) + + +@pytest.fixture +def fake_db_for_migration(fake_metadata_resolver): + content = {} + add_package( + content, + fake_metadata_resolver, + 'docker-database', + 'latest', + description='SONiC database service', + default_reference='1.0.0', + installed=True, + built_in=True + ) + add_package( + content, + fake_metadata_resolver, + 'docker-orchagent', + 'latest', + description='SONiC switch state service', + default_reference='1.0.0', + installed=True, + built_in=True + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test', + '1.6.0', + description='SONiC Package Manager Test Package', + default_reference='1.6.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-2', + '2.0.0', + description='SONiC Package Manager Test Package #2', + default_reference='2.0.0', + installed=False, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-3', + '1.6.0', + description='SONiC Package Manager Test Package #3', + default_reference='1.6.0', + installed=True, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-4', + '1.5.0', + description='SONiC Package Manager Test Package #4', + default_reference='1.5.0', + installed=True, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-5', + '1.5.0', + description='SONiC Package Manager Test Package #5', + default_reference='1.5.0', + installed=True, + built_in=False + ) + add_package( + content, + fake_metadata_resolver, + 'Azure/docker-test-6', + '2.0.0', + description='SONiC Package Manager Test Package #6', + default_reference='2.0.0', + installed=True, + built_in=False + ) + + yield PackageDatabase(content) + + +@pytest.fixture() +def sonic_fs(fs): + fs.create_file('/proc/1/root') + fs.create_dir(ETC_SONIC_PATH) + fs.create_dir(SYSTEMD_LOCATION) + fs.create_dir(DOCKER_CTL_SCRIPT_LOCATION) + fs.create_dir(SERVICE_MGMT_SCRIPT_LOCATION) + fs.create_dir(MONIT_CONF_LOCATION) + fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_FILE_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, TIMER_UNIT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_MGMT_SCRIPT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, DOCKER_CTL_SCRIPT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, MONIT_CONF_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, DEBUG_DUMP_SCRIPT_TEMPLATE)) + yield fs + + +@pytest.fixture(autouse=True) +def patch_pkgutil(): + with mock.patch('pkgutil.get_loader'): + yield + + +@pytest.fixture +def package_manager(mock_docker_api, + mock_registry_resolver, + mock_service_creator, + fake_metadata_resolver, + fake_db, + fake_device_info): + yield PackageManager(mock_docker_api, mock_registry_resolver, + fake_db, fake_metadata_resolver, + mock_service_creator, + fake_device_info, + MagicMock()) + + +@pytest.fixture +def anything(): + """ Fixture that returns Any object that can be used in + assert_called_*_with to match any object passed. """ + + class Any: + def __eq__(self, other): + return True + + yield Any() diff --git a/tests/sonic_package_manager/test_cli.py b/tests/sonic_package_manager/test_cli.py new file mode 100644 index 0000000000..695d8cba58 --- /dev/null +++ b/tests/sonic_package_manager/test_cli.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +from click.testing import CliRunner + +from sonic_package_manager import main + + +def test_show_changelog(package_manager, fake_metadata_resolver): + """ Test case for "sonic-package-manager package show changelog [NAME]" """ + + runner = CliRunner() + changelog = { + "1.0.0": { + "changes": ["Initial release"], + "author": "Stepan Blyshchak", + "email": "stepanb@nvidia.com", + "date": "Mon, 25 May 2020 12:24:30 +0300" + }, + "1.1.0": { + "changes": [ + "Added functionality", + "Bug fixes" + ], + "author": "Stepan Blyshchak", + "email": "stepanb@nvidia.com", + "date": "Fri, 23 Oct 2020 12:26:08 +0300" + } + } + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['changelog'] = changelog + + expected_output = """\ +1.0.0: + + • Initial release + + Stepan Blyshchak (stepanb@nvidia.com) Mon, 25 May 2020 12:24:30 +0300 + +1.1.0: + + • Added functionality + • Bug fixes + + Stepan Blyshchak (stepanb@nvidia.com) Fri, 23 Oct 2020 12:26:08 +0300 + +""" + + result = runner.invoke(main.show.commands['package'].commands['changelog'], + ['test-package'], obj=package_manager) + + assert result.exit_code == 0 + assert result.output == expected_output + + +def test_show_changelog_no_changelog(package_manager): + """ Test case for "sonic-package-manager package show changelog [NAME]" + when there is no changelog provided by package. """ + + runner = CliRunner() + result = runner.invoke(main.show.commands['package'].commands['changelog'], ['test-package'], obj=package_manager) + + assert result.exit_code == 1 + assert result.output == 'Failed to print package changelog: No changelog for package test-package\n' diff --git a/tests/sonic_package_manager/test_constraint.py b/tests/sonic_package_manager/test_constraint.py new file mode 100644 index 0000000000..2e7067ef63 --- /dev/null +++ b/tests/sonic_package_manager/test_constraint.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +from sonic_package_manager import version +from sonic_package_manager.constraint import PackageConstraint +from sonic_package_manager.version import Version, VersionRange + + +def test_constraint(): + package_constraint = PackageConstraint.parse('swss>1.0.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('0.9.1')) + assert package_constraint.constraint.allows(Version.parse('1.1.1')) + + +def test_constraint_range(): + package_constraint = PackageConstraint.parse('swss^1.2.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.5')) + assert not package_constraint.constraint.allows(Version.parse('2.0.1')) + + +def test_constraint_strict(): + package_constraint = PackageConstraint.parse('swss==1.2.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.0')) + + +def test_constraint_match(): + package_constraint = PackageConstraint.parse('swss==1.2*.*') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.0')) + + +def test_constraint_multiple(): + package_constraint = PackageConstraint.parse('swss>1.2.0,<3.0.0,!=2.2.2') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('2.2.2')) + assert not package_constraint.constraint.allows(Version.parse('3.2.0')) + assert not package_constraint.constraint.allows(Version.parse('0.2.0')) + assert package_constraint.constraint.allows(Version.parse('2.2.3')) + assert package_constraint.constraint.allows(Version.parse('1.2.3')) + + +def test_constraint_only_name(): + package_constraint = PackageConstraint.parse('swss') + assert package_constraint.name == 'swss' + assert package_constraint.constraint == VersionRange() + + +def test_constraint_from_dict(): + package_constraint = PackageConstraint.parse({ + 'name': 'swss', + 'version': '^1.0.0', + 'components': { + 'libswsscommon': '^1.1.0', + }, + }) + assert package_constraint.name == 'swss' + assert package_constraint.constraint.allows(Version.parse('1.0.0')) + assert not package_constraint.constraint.allows(Version.parse('2.0.0')) + assert package_constraint.components['libswsscommon'].allows(Version.parse('1.2.0')) + assert not package_constraint.components['libswsscommon'].allows(Version.parse('1.0.0')) + assert not package_constraint.components['libswsscommon'].allows(Version.parse('2.0.0')) + + +def test_version_to_tag(): + assert version.version_to_tag(Version.parse('1.0.0-rc0')) == '1.0.0-rc0' + assert version.version_to_tag(Version.parse('1.0.0-rc0+152')) == '1.0.0-rc0_152' + + +def test_tag_to_version(): + assert str(version.tag_to_version('1.0.0-rc0_152')) == '1.0.0-rc0+152' + assert str(version.tag_to_version('1.0.0-rc0')) == '1.0.0-rc0' diff --git a/tests/sonic_package_manager/test_database.py b/tests/sonic_package_manager/test_database.py new file mode 100644 index 0000000000..1c565d6f4c --- /dev/null +++ b/tests/sonic_package_manager/test_database.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +import pytest + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.errors import ( + PackageNotFoundError, + PackageAlreadyExistsError, + PackageManagerError +) +from sonic_package_manager.version import Version + + +def test_database_get_package(fake_db): + swss_package = fake_db.get_package('swss') + assert swss_package.installed + assert swss_package.built_in + assert swss_package.repository == 'docker-orchagent' + assert swss_package.default_reference == '1.0.0' + assert swss_package.version == Version(1, 0, 0) + + +def test_database_get_package_not_builtin(fake_db): + test_package = fake_db.get_package('test-package') + assert not test_package.installed + assert not test_package.built_in + assert test_package.repository == 'Azure/docker-test' + assert test_package.default_reference == '1.6.0' + assert test_package.version is None + + +def test_database_get_package_not_existing(fake_db): + with pytest.raises(PackageNotFoundError): + fake_db.get_package('abc') + + +def test_database_add_package(fake_db): + fake_db.add_package('test-package-99', 'Azure/docker-test-99') + test_package = fake_db.get_package('test-package-99') + assert not test_package.installed + assert not test_package.built_in + assert test_package.repository == 'Azure/docker-test-99' + assert test_package.default_reference is None + assert test_package.version is None + + +def test_database_add_package_existing(fake_db): + with pytest.raises(PackageAlreadyExistsError): + fake_db.add_package('swss', 'Azure/docker-orchagent') + + +def test_database_update_package(fake_db): + test_package = fake_db.get_package('test-package-2') + test_package.installed = True + test_package.version = Version(1, 2, 3) + fake_db.update_package(test_package) + test_package = fake_db.get_package('test-package-2') + assert test_package.installed + assert test_package.version == Version(1, 2, 3) + + +def test_database_update_package_non_existing(fake_db): + test_package = PackageEntry('abc', 'abc') + with pytest.raises(PackageNotFoundError): + fake_db.update_package(test_package) + + +def test_database_remove_package(fake_db): + fake_db.remove_package('test-package') + assert not fake_db.has_package('test-package') + + +def test_database_remove_package_non_existing(fake_db): + with pytest.raises(PackageNotFoundError): + fake_db.remove_package('non-existing-package') + + +def test_database_remove_package_installed(fake_db): + with pytest.raises(PackageManagerError, + match='Package test-package-3 is installed, ' + 'uninstall it first'): + fake_db.remove_package('test-package-3') + + +def test_database_remove_package_built_in(fake_db): + with pytest.raises(PackageManagerError, + match='Package swss is built-in, ' + 'cannot remove it'): + fake_db.remove_package('swss') diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py new file mode 100644 index 0000000000..256832118e --- /dev/null +++ b/tests/sonic_package_manager/test_manager.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python +from unittest.mock import Mock, call + +import pytest + +from sonic_package_manager.errors import * +from sonic_package_manager.version import Version + + +def test_installation_not_installed(package_manager): + package_manager.install('test-package') + + +def test_installation_already_installed(package_manager): + with pytest.raises(PackageInstallationError, + match='swss is already installed'): + package_manager.install('swss') + + +def test_installation_dependencies(package_manager, fake_metadata_resolver, mock_docker_api): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['depends'] = ['swss^2.0.0'] + with pytest.raises(PackageInstallationError, + match='Package test-package requires swss>=2.0.0,<3.0.0 ' + 'but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_dependencies_missing_package(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['depends'] = ['missing-package>=1.0.0'] + with pytest.raises(PackageInstallationError, + match='Package test-package requires ' + 'missing-package>=1.0.0 but it is not installed'): + package_manager.install('test-package') + + +def test_installation_dependencies_satisfied(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['depends'] = ['database>=1.0.0', 'swss>=1.0.0'] + package_manager.install('test-package') + + +def test_installation_components_dependencies_satisfied(package_manager, fake_metadata_resolver): + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + manifest = metadata['manifest'] + metadata['components'] = { + 'libswsscommon': Version.parse('1.1.0') + } + manifest['package']['depends'] = [ + { + 'name': 'swss', + 'version': '>=1.0.0', + 'components': { + 'libswsscommon': '^1.0.0', + }, + }, + ] + package_manager.install('test-package') + + +def test_installation_components_dependencies_not_satisfied(package_manager, fake_metadata_resolver): + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + manifest = metadata['manifest'] + metadata['components'] = { + 'libswsscommon': Version.parse('1.1.0') + } + manifest['package']['depends'] = [ + { + 'name': 'swss', + 'version': '>=1.0.0', + 'components': { + 'libswsscommon': '^1.1.0', + }, + }, + ] + with pytest.raises(PackageInstallationError, + match='Package test-package requires libswsscommon >=1.1.0,<2.0.0 ' + 'in package swss>=1.0.0 but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_components_dependencies_implicit(package_manager, fake_metadata_resolver): + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + manifest = metadata['manifest'] + metadata['components'] = { + 'libswsscommon': Version.parse('2.1.0') + } + manifest['package']['depends'] = [ + { + 'name': 'swss', + 'version': '>=1.0.0', + }, + ] + with pytest.raises(PackageInstallationError, + match='Package test-package requires libswsscommon >=2.1.0,<3.0.0 ' + 'in package swss>=1.0.0 but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_components_dependencies_explicitely_allowed(package_manager, fake_metadata_resolver): + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + manifest = metadata['manifest'] + metadata['components'] = { + 'libswsscommon': Version.parse('2.1.0') + } + manifest['package']['depends'] = [ + { + 'name': 'swss', + 'version': '>=1.0.0', + 'components': { + 'libswsscommon': '>=1.0.0,<3.0.0' + } + }, + ] + package_manager.install('test-package') + + +def test_installation_breaks(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['breaks'] = ['swss^1.0.0'] + with pytest.raises(PackageInstallationError, + match='Package test-package conflicts with ' + 'swss>=1.0.0,<2.0.0 but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_breaks_missing_package(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['breaks'] = ['missing-package^1.0.0'] + package_manager.install('test-package') + + +def test_installation_breaks_not_installed_package(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['breaks'] = ['test-package-2^1.0.0'] + package_manager.install('test-package') + + +def test_installation_base_os_constraint(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['base-os']['libswsscommon'] = '>=2.0.0' + with pytest.raises(PackageSonicRequirementError, + match='Package test-package requires base OS component libswsscommon ' + 'version >=2.0.0 while the installed version is 1.0.0'): + package_manager.install('test-package') + + +def test_installation_base_os_constraint_satisfied(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['package']['base-os']['libswsscommon'] = '>=1.0.0' + package_manager.install('test-package') + + +def test_installation_cli_plugin(package_manager, fake_metadata_resolver, anything): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['cli']= {'show': '/cli/plugin.py'} + package_manager._install_cli_plugins = Mock() + package_manager.install('test-package') + package_manager._install_cli_plugins.assert_called_once_with(anything) + + +def test_installation_cli_plugin_skipped(package_manager, fake_metadata_resolver, anything): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['cli']= {'show': '/cli/plugin.py'} + package_manager._install_cli_plugins = Mock() + package_manager.install('test-package', skip_cli_plugin_installation=True) + package_manager._install_cli_plugins.assert_not_called() + + +def test_installation_cli_plugin_is_mandatory_but_skipped(package_manager, fake_metadata_resolver): + manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] + manifest['cli']= {'mandatory': True} + with pytest.raises(PackageManagerError, + match='CLI is mandatory for package test-package but ' + 'it was requested to be not installed'): + package_manager.install('test-package', skip_cli_plugin_installation=True) + + +def test_installation(package_manager, mock_docker_api, anything): + package_manager.install('test-package') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', '1.6.0') + + +def test_installation_using_reference(package_manager, + fake_metadata_resolver, + mock_docker_api, + anything): + ref = 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + fake_metadata_resolver.metadata_store['Azure/docker-test'][ref] = metadata + + package_manager.install(f'test-package@{ref}') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', f'{ref}') + + +def test_manager_installation_tag(package_manager, + mock_docker_api, + anything): + package_manager.install(f'test-package==1.6.0') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', '1.6.0') + + +def test_installation_from_file(package_manager, mock_docker_api, sonic_fs): + sonic_fs.create_file('Azure/docker-test:1.6.0') + package_manager.install(tarball='Azure/docker-test:1.6.0') + mock_docker_api.load.assert_called_once_with('Azure/docker-test:1.6.0') + + +def test_installation_from_registry(package_manager, mock_docker_api): + package_manager.install(repotag='Azure/docker-test:1.6.0') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', '1.6.0') + + +def test_installation_from_registry_using_digest(package_manager, mock_docker_api, fake_metadata_resolver): + ref = 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + metadata = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0'] + fake_metadata_resolver.metadata_store['Azure/docker-test'][ref] = metadata + + ref = 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + package_manager.install(repotag=f'Azure/docker-test@{ref}') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', ref) + + +def test_installation_from_file_known_package(package_manager, fake_db, sonic_fs): + repository = fake_db.get_package('test-package').repository + sonic_fs.create_file('Azure/docker-test:1.6.0') + package_manager.install(tarball='Azure/docker-test:1.6.0') + # locally installed package does not override already known package repository + assert repository == fake_db.get_package('test-package').repository + + +def test_installation_from_file_unknown_package(package_manager, fake_db, sonic_fs): + assert not fake_db.has_package('test-package-4') + sonic_fs.create_file('Azure/docker-test-4:1.5.0') + package_manager.install(tarball='Azure/docker-test-4:1.5.0') + assert fake_db.has_package('test-package-4') + + +def test_upgrade_from_file_known_package(package_manager, fake_db, sonic_fs): + repository = fake_db.get_package('test-package-6').repository + # install older version from repository + package_manager.install('test-package-6==1.5.0') + # upgrade from file + sonic_fs.create_file('Azure/docker-test-6:2.0.0') + package_manager.upgrade(tarball='Azure/docker-test-6:2.0.0') + # locally installed package does not override already known package repository + assert repository == fake_db.get_package('test-package-6').repository + + +def test_installation_non_default_owner(package_manager, anything, mock_service_creator): + package_manager.install('test-package', default_owner='kube') + mock_service_creator.create.assert_called_once_with(anything, state='disabled', owner='kube') + + +def test_installation_enabled(package_manager, anything, mock_service_creator): + package_manager.install('test-package', enable=True) + mock_service_creator.create.assert_called_once_with(anything, state='enabled', owner='local') + + +def test_installation_fault(package_manager, mock_docker_api, mock_service_creator): + # make 'tag' to fail + mock_service_creator.create = Mock(side_effect=Exception('Failed to create service')) + # 'rmi' is called on rollback + mock_docker_api.rmi = Mock(side_effect=Exception('Failed to remove image')) + # assert that the rollback does not hide the original failure. + with pytest.raises(Exception, match='Failed to create service'): + package_manager.install('test-package') + mock_docker_api.rmi.assert_called_once() + + +def test_installation_package_with_description(package_manager, fake_metadata_resolver): + package_entry = package_manager.database.get_package('test-package') + description = package_entry.description + references = fake_metadata_resolver.metadata_store[package_entry.repository] + manifest = references[package_entry.default_reference]['manifest'] + new_description = description + ' changed description ' + manifest['package']['description'] = new_description + package_manager.install('test-package') + package_entry = package_manager.database.get_package('test-package') + description = package_entry.description + assert description == new_description + + +def test_manager_installation_version_range(package_manager): + with pytest.raises(PackageManagerError, + match='Can only install specific version. ' + 'Use only following expression "test-package==" ' + 'to install specific version'): + package_manager.install(f'test-package>=1.6.0') + + +def test_manager_upgrade(package_manager, sonic_fs): + package_manager.install('test-package-6==1.5.0') + package_manager.upgrade('test-package-6==2.0.0') + + upgraded_package = package_manager.get_installed_package('test-package-6') + assert upgraded_package.entry.version == Version(2, 0, 0) + + +def test_manager_migration(package_manager, fake_db_for_migration): + package_manager.install = Mock() + package_manager.upgrade = Mock() + package_manager.migrate_packages(fake_db_for_migration) + + # test-package-3 was installed but there is a newer version installed + # in fake_db_for_migration, asserting for upgrade + package_manager.upgrade.assert_has_calls([call('test-package-3==1.6.0')], any_order=True) + + package_manager.install.assert_has_calls([ + # test-package-4 was not present in DB at all, but it is present and installed in + # fake_db_for_migration, thus asserting that it is going to be installed. + call('test-package-4==1.5.0'), + # test-package-5 1.5.0 was installed in fake_db_for_migration but the default + # in current db is 1.9.0, assert that migration will install the newer version. + call('test-package-5==1.9.0'), + # test-package-6 2.0.0 was installed in fake_db_for_migration but the default + # in current db is 1.5.0, assert that migration will install the newer version. + call('test-package-6==2.0.0')], + any_order=True + ) diff --git a/tests/sonic_package_manager/test_manifest.py b/tests/sonic_package_manager/test_manifest.py new file mode 100644 index 0000000000..f6028e0437 --- /dev/null +++ b/tests/sonic_package_manager/test_manifest.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +import pytest + +from sonic_package_manager.manifest import Manifest, ManifestError +from sonic_package_manager.version import VersionRange + + +def test_manifest_v1_defaults(): + manifest = Manifest.marshal({'package': {'name': 'test', + 'version': '1.0.0'}, + 'service': {'name': 'test'}}) + assert manifest['package']['depends'] == [] + assert manifest['package']['breaks'] == [] + assert manifest['package']['base-os'] == dict() + assert not manifest['service']['asic-service'] + assert manifest['service']['host-service'] + + +def test_manifest_v1_invalid_version(): + with pytest.raises(ManifestError): + Manifest.marshal({'package': {'version': 'abc', 'name': 'test'}, + 'service': {'name': 'test'}}) + + +def test_manifest_v1_invalid_package_constraint(): + with pytest.raises(ManifestError): + Manifest.marshal({'package': {'name': 'test', 'version': '1.0.0', + 'depends': ['swss>a']}, + 'service': {'name': 'test'}}) + + +def test_manifest_v1_service_spec(): + manifest = Manifest.marshal({'package': {'name': 'test', + 'version': '1.0.0'}, + 'service': {'name': 'test', 'asic-service': True}}) + assert manifest['service']['asic-service'] + + +def test_manifest_v1_mounts(): + manifest = Manifest.marshal({'version': '1.0.0', 'package': {'name': 'test', + 'version': '1.0.0'}, + 'service': {'name': 'cpu-report'}, + 'container': {'privileged': True, + 'mounts': [{'source': 'a', 'target': 'b', 'type': 'bind'}]}}) + assert manifest['container']['mounts'][0]['source'] == 'a' + assert manifest['container']['mounts'][0]['target'] == 'b' + assert manifest['container']['mounts'][0]['type'] == 'bind' + + +def test_manifest_v1_mounts_invalid(): + with pytest.raises(ManifestError): + Manifest.marshal({'version': '1.0.0', 'package': {'name': 'test', 'version': '1.0.0'}, + 'service': {'name': 'cpu-report'}, + 'container': {'privileged': True, + 'mounts': [{'not-source': 'a', 'target': 'b', 'type': 'bind'}]}}) + + +def test_manifest_v1_unmarshal(): + manifest_json_input = {'package': {'name': 'test', 'version': '1.0.0', + 'depends': ['swss>1.0.0']}, + 'service': {'name': 'test'}} + manifest = Manifest.marshal(manifest_json_input) + manifest_json = manifest.unmarshal() + for key, section in manifest_json_input.items(): + for field, value in section.items(): + assert manifest_json[key][field] == value diff --git a/tests/sonic_package_manager/test_metadata.py b/tests/sonic_package_manager/test_metadata.py new file mode 100644 index 0000000000..4636c18282 --- /dev/null +++ b/tests/sonic_package_manager/test_metadata.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +import contextlib +from unittest.mock import Mock, MagicMock + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.errors import MetadataError +from sonic_package_manager.metadata import MetadataResolver +from sonic_package_manager.version import Version + + +def test_metadata_resolver_local(mock_registry_resolver, mock_docker_api): + metadata_resolver = MetadataResolver(mock_docker_api, mock_registry_resolver) + # it raises exception because mock manifest is not a valid manifest + # but this is not a test objective, so just suppress the error. + with contextlib.suppress(MetadataError): + metadata_resolver.from_local('image') + mock_docker_api.labels.assert_called_once() + + +def test_metadata_resolver_remote(mock_registry_resolver, mock_docker_api): + metadata_resolver = MetadataResolver(mock_docker_api, mock_registry_resolver) + mock_registry = MagicMock() + mock_registry.manifest = MagicMock(return_value={'config': {'digest': 'some-digest'}}) + + def return_mock_registry(repository): + return mock_registry + + mock_registry_resolver.get_registry_for = Mock(side_effect=return_mock_registry) + # it raises exception because mock manifest is not a valid manifest + # but this is not a test objective, so just suppress the error. + with contextlib.suppress(MetadataError): + metadata_resolver.from_registry('test-repository', '1.2.0') + mock_registry_resolver.get_registry_for.assert_called_once_with('test-repository') + mock_registry.manifest.assert_called_once_with('test-repository', '1.2.0') + mock_registry.blobs.assert_called_once_with('test-repository', 'some-digest') + mock_docker_api.labels.assert_not_called() diff --git a/tests/sonic_package_manager/test_reference.py b/tests/sonic_package_manager/test_reference.py new file mode 100644 index 0000000000..c986632c43 --- /dev/null +++ b/tests/sonic_package_manager/test_reference.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import pytest + +from sonic_package_manager.reference import PackageReference + + +def test_reference(): + package_constraint = PackageReference.parse( + 'swss@sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + ) + assert package_constraint.name == 'swss' + assert package_constraint.reference == 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + + +def test_reference_invalid(): + with pytest.raises(ValueError): + PackageReference.parse('swssfdsf') diff --git a/tests/sonic_package_manager/test_registry.py b/tests/sonic_package_manager/test_registry.py new file mode 100644 index 0000000000..0d82499df3 --- /dev/null +++ b/tests/sonic_package_manager/test_registry.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +from sonic_package_manager.registry import RegistryResolver + + +def test_get_registry_for(): + resolver = RegistryResolver() + registry = resolver.get_registry_for('debian') + assert registry is resolver.DockerHubRegistry + registry = resolver.get_registry_for('Azure/sonic') + assert registry is resolver.DockerHubRegistry + registry = resolver.get_registry_for('registry-server:5000/docker') + assert registry.url == 'https://registry-server:5000' + registry = resolver.get_registry_for('registry-server.com/docker') + assert registry.url == 'https://registry-server.com' diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py new file mode 100644 index 0000000000..e69b4447f8 --- /dev/null +++ b/tests/sonic_package_manager/test_service_creator.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +import os +from unittest.mock import Mock, MagicMock + +import pytest + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.metadata import Metadata +from sonic_package_manager.package import Package +from sonic_package_manager.service_creator.creator import ( + ServiceCreator, + ETC_SONIC_PATH, DOCKER_CTL_SCRIPT_LOCATION, + SERVICE_MGMT_SCRIPT_LOCATION, SYSTEMD_LOCATION, MONIT_CONF_LOCATION, DEBUG_DUMP_SCRIPT_LOCATION +) +from sonic_package_manager.service_creator.feature import FeatureRegistry + + +@pytest.fixture +def manifest(): + return Manifest.marshal({ + 'package': { + 'name': 'test', + 'version': '1.0.0', + }, + 'service': { + 'name': 'test', + 'requires': ['database'], + 'after': ['database', 'swss', 'syncd'], + 'before': ['ntp-config'], + 'dependent-of': ['swss'], + 'asic-service': False, + 'host-service': True, + }, + 'container': { + 'privileged': True, + 'volumes': [ + '/etc/sonic:/etc/sonic:ro' + ] + } + }) + + +def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) + assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) + assert sonic_fs.exists(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, 'test.sh')) + assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) + assert sonic_fs.exists(os.path.join(MONIT_CONF_LOCATION, 'monit_test')) + + +def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert not sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) + + manifest['service']['delayed'] = True + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) + + +def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): + creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + + manifest['package']['debug-dump'] = '/some/command' + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + + +def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): + mock_table = Mock() + mock_table.get = Mock(return_value=(True, (('field_2', 'original_value_2'),))) + mock_sonic_db.initial_table = Mock(return_value=mock_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_table) + mock_sonic_db.running_table = Mock(return_value=mock_table) + + creator = ServiceCreator(mock_feature_registry, mock_sonic_db) + + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, Metadata(manifest)) + creator.create(package) + + assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + + manifest['package']['init-cfg'] = { + 'TABLE_A': { + 'key_a': { + 'field_1': 'value_1', + 'field_2': 'value_2' + }, + }, + } + package = Package(entry, Metadata(manifest)) + + creator.create(package) + + mock_table.set.assert_called_with('key_a', [('field_1', 'value_1'), + ('field_2', 'original_value_2')]) + + +def test_feature_registration(mock_sonic_db, manifest): + mock_feature_table = Mock() + mock_feature_table.get = Mock(return_value=(False, ())) + mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) + mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + feature_registry = FeatureRegistry(mock_sonic_db) + feature_registry.register(manifest) + mock_feature_table.set.assert_called_with('test', [ + ('state', 'disabled'), + ('auto_restart', 'enabled'), + ('high_mem_alert', 'disabled'), + ('set_owner', 'local'), + ('has_per_asic_scope', 'False'), + ('has_global_scope', 'True'), + ('has_timer', 'False'), + ]) + + +def test_feature_registration_with_timer(mock_sonic_db, manifest): + manifest['service']['delayed'] = True + mock_feature_table = Mock() + mock_feature_table.get = Mock(return_value=(False, ())) + mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) + mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + feature_registry = FeatureRegistry(mock_sonic_db) + feature_registry.register(manifest) + mock_feature_table.set.assert_called_with('test', [ + ('state', 'disabled'), + ('auto_restart', 'enabled'), + ('high_mem_alert', 'disabled'), + ('set_owner', 'local'), + ('has_per_asic_scope', 'False'), + ('has_global_scope', 'True'), + ('has_timer', 'True'), + ]) + + +def test_feature_registration_with_non_default_owner(mock_sonic_db, manifest): + mock_feature_table = Mock() + mock_feature_table.get = Mock(return_value=(False, ())) + mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) + mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + feature_registry = FeatureRegistry(mock_sonic_db) + feature_registry.register(manifest, owner='kube') + mock_feature_table.set.assert_called_with('test', [ + ('state', 'disabled'), + ('auto_restart', 'enabled'), + ('high_mem_alert', 'disabled'), + ('set_owner', 'kube'), + ('has_per_asic_scope', 'False'), + ('has_global_scope', 'True'), + ('has_timer', 'False'), + ]) diff --git a/tests/sonic_package_manager/test_utils.py b/tests/sonic_package_manager/test_utils.py new file mode 100644 index 0000000000..c4d8b15840 --- /dev/null +++ b/tests/sonic_package_manager/test_utils.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +from sonic_package_manager import utils + + +def test_make_python_identifier(): + assert utils.make_python_identifier('-some-package name').isidentifier() + assert utils.make_python_identifier('01 leading digit').isidentifier() From 06dfd3002166ed072ab345949f57f5e24e21e7a0 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 23 Mar 2021 14:30:27 +0200 Subject: [PATCH 09/60] add help for some options Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index dcc048079c..af0a11d60a 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -91,7 +91,7 @@ def handle_parse_result(self, ctx, opts, args): PACKAGE_SOURCE_OPTIONS = [ click.option('--from-repository', - help='Install package directly from image registry repository', + help='Fetch package directly from image registry repository', cls=MutuallyExclusiveOption, mutually_exclusive=['from_tarball', 'package_expr']), click.option('--from-tarball', @@ -99,7 +99,7 @@ def handle_parse_result(self, ctx, opts, args): readable=True, file_okay=True, dir_okay=False), - help='Install package from saved image tarball', + help='Fetch package from saved image tarball', cls=MutuallyExclusiveOption, mutually_exclusive=['from_repository', 'package_expr']), click.argument('package-expr', @@ -281,8 +281,8 @@ def changelog(ctx, @repository.command() @click.argument('name', type=str) @click.argument('repository', type=str) -@click.option('--default-reference', type=str) -@click.option('--description', type=str) +@click.option('--default-reference', type=str, help='Default installation reference.') +@click.option('--description', type=str, help='Default installation reference.') @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): From 725210d908106e45ded934969420bfc8e163e942 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 23 Mar 2021 14:30:43 +0200 Subject: [PATCH 10/60] add command line reference for sonic-package-manager Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 294 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 293 insertions(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index ffbc0c26f4..b55e36d53d 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -143,6 +143,7 @@ * [Watermark Show commands](#watermark-show-commands) * [Watermark Config commands](#watermark-config-commands) * [Software Installation and Management](#software-installation-and-management) + * [SONiC Package Manager](#sonic-package-manager) * [SONiC Installer](#sonic-installer) * [Troubleshooting Commands](#troubleshooting-commands) * [Routing Stack](#routing-stack) @@ -7961,8 +7962,292 @@ Go Back To [Beginning of the document](#) or [Beginning of this section](#waterm ## Software Installation and Management -SONiC software can be installed in two methods, viz, "using sonic-installer tool", "ONIE Installer". +SONiC software image can be installed in two methods, viz, "using sonic-installer tool", "ONIE Installer". +SONiC feature Docker images (aka "SONiC packages") available to be installed with *sonic-package-manager* utility. + +### SONiC Package Manager + +This is a command line tool that provides functionality to manage SONiC Packages on SONiC device. + +**sonic-package-manager list** + +This command lists all available SONiC packages, their desription, installed version and installation status. +SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-In" status means that a feature is built-in to SONiC image and can't be upgraded or uninstalled. + +- Usage: + ``` + sonic-package-manager list + ``` + +- Example: + ``` + admin@sonic:~$ sonic-package-manager list + Name Repository Description Version Status + -------------- --------------------------- ---------------------------- --------- --------- + database docker-database SONiC database package 1.0.0 Built-In + dhcp-relay docker-dhcp-relay SONiC dhcp-relay package 1.0.0 Installed + fpm-frr docker-fpm-frr SONiC fpm-frr package 1.0.0 Built-In + lldp docker-lldp SONiC lldp package 1.0.0 Built-In + macsec docker-macsec SONiC macsec package 1.0.0 Built-In + mgmt-framework docker-sonic-mgmt-framework SONiC mgmt-framework package 1.0.0 Built-In + nat docker-nat SONiC nat package 1.0.0 Built-In + pmon docker-platform-monitor SONiC pmon package 1.0.0 Built-In + radv docker-router-advertiser SONiC radv package 1.0.0 Built-In + sflow docker-sflow SONiC sflow package 1.0.0 Built-In + snmp docker-snmp SONiC snmp package 1.0.0 Built-In + swss docker-orchagent SONiC swss package 1.0.0 Built-In + syncd docker-syncd-mlnx SONiC syncd package 1.0.0 Built-In + teamd docker-teamd SONiC teamd package 1.0.0 Built-In + telemetry docker-sonic-telemetry SONiC telemetry package 1.0.0 Built-In + ``` + +**sonic-package-manager repository add** + +This command will add a new entry in the package database. The package has to be *Not Installed* in order to be removed from package database. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY + + Add a new repository to database. + + Options: + --default-reference TEXT Default installation reference. + --description TEXT Optional package entry description. + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager repository add \ + cpu-report azure/sonic-cpu-report --default-reference 1.0.0 + ``` + +**sonic-package-manager repository remove** + +This command will remove an entry from the package database. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager repository remove [OPTIONS] NAME + + Remove package from database. + + Options: + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager repository remove cpu-report + ``` + +**sonic-package-manager install** + +This command pulls and installs package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] + + Install package + + Options: + --enable Set the default state of the feature to + enabled and enable feature right after + installation. NOTE: user needs to execute + "config save -y" to make this setting + persistent + + --default-owner [local|kube] Default owner configuration setting for a + feature + + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually + exclusive with arguments: [from_tarball, + package_expr]. + + --from-tarball FILE Fetch package from saved image tarball + NOTE: This argument is mutually exclusive + with arguments: [from_repository, + package_expr]. + + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or + DEBUG + + --skip-cli-plugin-installation Do not install CLI plugins provided by the + package on the host OS. NOTE: In case when + package /cli/mandatory field is set to True + this option will fail the installation. + + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager install dhcp-relay==1.0.2 + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager install --from-repository azure/sonic-cpu-report:latest + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager install --from-tarball sonic-docker-image.gz + ``` + +**sonic-package-manager uninstall** + +This command uninstalls package from SONiC host. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager uninstall [OPTIONS] NAME + + Uninstall package + + Options: + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --help Show this message and exit. + + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager uninstall dhcp-relay + ``` + +**sonic-package-manager upgrade** + +This command upgrades package on SONiC host to a newer version. The procedure of upgrading a package will restart the corresponding service. *NOTE*: this command requires elevated (root) privileges to run. + +- Usage: + ``` + Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] + + Upgrade package + + Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually + exclusive with arguments: [package_expr, + from_tarball]. + + --from-tarball FILE Fetch package from saved image tarball + NOTE: This argument is mutually exclusive + with arguments: [package_expr, + from_repository]. + + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or + DEBUG + + --skip-cli-plugin-installation Do not install CLI plugins provided by the + package on the host OS. NOTE: In case when + package /cli/mandatory field is set to True + this option will fail the installation. + + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager upgrade dhcp-relay==2.0.0 + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager upgrade --from-repository azure/sonic-cpu-report:latest + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager upgrade --from-tarball sonic-docker-image.gz + ``` + +**sonic-package-manager show package versions** + +This command will access repository for corresponding package and retrieve a list of available versions. + +- Usage: + ``` + Usage: sonic-package-manager show package versions [OPTIONS] NAME + + Print available versions + + Options: + --all Show all available tags in repository + --plain Plain output + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sonic-package-manager show package versions dhcp-relay + • 1.0.0 + • 1.0.2 + • 2.0.0 + ``` + +**sonic-package-manager show package changelog** + +This command fetches the changelog from package manifest and displays it. *NOTE*: package changelog can be retrieved from registry or read from image tarball without installing it. + +- Usage: + ``` + Usage: sonic-package-manager show package changelog [OPTIONS] [PACKAGE_EXPR] + + Print package changelog + + Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually exclusive + with arguments: [from_tarball, package_expr]. + --from-tarball FILE Fetch package from saved image tarball NOTE: This + argument is mutually exclusive with arguments: + [package_expr, from_repository]. + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sonic-package-manager show package changelog dhcp-relay + 1.0.0: + + • Initial release + + Author (author@email.com) Mon, 25 May 2020 12:25:00 +0300 + ``` + +**sonic-package-manager show package manifest** + +This command fetches the package manifest and displays it. *NOTE*: package manifest can be retrieved from registry or read from image tarball without installing it. + +- Usage: + ``` + Usage: sonic-package-manager show package manifest [OPTIONS] [PACKAGE_EXPR] + + Print package manifest content + + Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually exclusive + with arguments: [package_expr, from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: This + argument is mutually exclusive with arguments: + [from_repository, package_expr]. + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sonic-package-manager show package manifest dhcp-relay==2.0.0 + { + "version": "1.0.0", + "package": { + "version": "2.0.0", + "depends": [ + "database>=1.0.0,<2.0.0" + ] + }, + "service": { + "name": "dhcp_relay" + } + } + ``` ### SONiC Installer This is a command line tool available as part of the SONiC software; If the device is already running the SONiC software, this tool can be used to install an alternate image in the partition. @@ -8033,6 +8318,13 @@ This command is used to install a new image on the alternate image partition. T Done ``` +SONiC image installation will install SONiC packages that are installed in currently running SONiC image. In order to perform clean SONiC installation use *--skip-package-migration* option when installing SONiC image: + +- Example: + ``` + admin@sonic:~$ sudo sonic-installer install https://sonic-jenkins.westus.cloudapp.azure.com/job/xxxx/job/buildimage-xxxx-all/xxx/artifact/target/sonic-xxxx.bin --skip-package-migration + ``` + **sonic-installer set_default** This command is be used to change the image which can be loaded by default in all the subsequent reboots. From f2ee39f27e16484defac5ca8af5fcd0dbbf5e009 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 23 Mar 2021 15:35:42 +0200 Subject: [PATCH 11/60] fix upgrade command missing argument Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index af0a11d60a..713113da61 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -378,7 +378,8 @@ def upgrade(ctx, from_repository, from_tarball, force, - yes): + yes, + skip_cli_plugin_installation): """ Upgrade package """ manager: PackageManager = ctx.obj From 7bf937ed5160bd08bca79fd5151c92e0771a2651 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 14:00:08 +0200 Subject: [PATCH 12/60] warm-reboot support for sonic packages Signed-off-by: Stepan Blyschak --- config/main.py | 12 +- scripts/fast-reboot | 119 ++++++++---------- scripts/generate_shutdown_order.py | 15 +++ setup.py | 2 + sonic_package_manager/manager.py | 23 +++- sonic_package_manager/manifest.py | 13 +- .../service_creator/creator.py | 63 +++++++++- tests/sonic_package_manager/conftest.py | 48 ++++++- tests/sonic_package_manager/test_manager.py | 4 +- .../test_service_creator.py | 29 ++++- 10 files changed, 241 insertions(+), 87 deletions(-) create mode 100644 scripts/generate_shutdown_order.py diff --git a/config/main.py b/config/main.py index d27562bd4e..5fb272a807 100644 --- a/config/main.py +++ b/config/main.py @@ -1833,20 +1833,28 @@ def warm_restart(ctx, redis_unix_socket_path): ctx.obj = {'db': config_db, 'state_db': state_db, 'prefix': prefix} @warm_restart.command('enable') -@click.argument('module', metavar='', default='system', required=False, type=click.Choice(["system", "swss", "bgp", "teamd"])) +@click.argument('module', metavar='', default='system', required=False) @click.pass_context def warm_restart_enable(ctx, module): state_db = ctx.obj['state_db'] + config_db = ctx.obj['db'] + feature_table = config_db.get_table('FEATURE') + if module != 'system' and module not in feature_table: + exit('Feature {} is unknown'.format(module)) prefix = ctx.obj['prefix'] _hash = '{}{}'.format(prefix, module) state_db.set(state_db.STATE_DB, _hash, 'enable', 'true') state_db.close(state_db.STATE_DB) @warm_restart.command('disable') -@click.argument('module', metavar='', default='system', required=False, type=click.Choice(["system", "swss", "bgp", "teamd"])) +@click.argument('module', metavar='', default='system', required=False) @click.pass_context def warm_restart_enable(ctx, module): state_db = ctx.obj['state_db'] + config_db = ctx.obj['db'] + feature_table = config_db.get_table('FEATURE') + if module != 'system' and module not in feature_table: + exit('Feature {} is unknown'.format(module)) prefix = ctx.obj['prefix'] _hash = '{}{}'.format(prefix, module) state_db.set(state_db.STATE_DB, _hash, 'enable', 'false') diff --git a/scripts/fast-reboot b/scripts/fast-reboot index 92648bd207..d11b143d56 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -7,6 +7,7 @@ WARM_DIR=/host/warmboot REDIS_FILE=dump.rdb REBOOT_SCRIPT_NAME=$(basename $0) REBOOT_TYPE="${REBOOT_SCRIPT_NAME}" +SHUTDOWN_ORDER_FILE="/etc/sonic/${REBOOT_TYPE}_order" VERBOSE=no FORCE=no IGNORE_ASIC=no @@ -547,82 +548,72 @@ if [ -x ${LOG_SSD_HEALTH} ]; then fi -# Kill nat docker after saving the conntrack table -debug "Stopping nat ..." -/usr/local/bin/dump_nat_entries.py -docker kill nat > /dev/null || true -systemctl stop nat -debug "Stopped nat ..." - -# Kill radv before stopping BGP service to prevent announcing our departure. -debug "Stopping radv service..." -systemctl stop radv -debug "Stopped radv service..." - -# Kill bgpd to start the bgp graceful restart procedure -debug "Stopping bgp ..." -systemctl stop bgp -debug "Stopped bgp ..." - -# Kill sflow docker -debug "Stopping sflow ..." -container kill sflow &> /dev/null || debug "Docker sflow is not running ($?) ..." -systemctl stop sflow -debug "Stopped sflow ..." - -# Kill lldp, otherwise it sends informotion about reboot. -# We call `docker kill lldp` to ensure the container stops as quickly as possible, -# then immediately call `systemctl stop lldp` to prevent the service from -# restarting the container automatically. -container kill lldp &> /dev/null || debug "Docker lldp is not running ($?) ..." -systemctl stop lldp - -if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then - debug "Stopping teamd ..." - systemctl stop teamd - debug "Stopped teamd ..." +if [[ -f ${SHUTDOWN_ORDER_FILE} ]]; then + SERVICE_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})" +else + # TODO: to be removed once sonic-buildimage change is in + if [[ "${REBOOT_TYPE}" == "fast-reboot" ]]; then + SERVICE_TO_STOP="nat radv bgp sflow lldp swss teamd syncd" + elif [[ "${REBOOT_TYPE}" == "fastfast-reboot" || "${REBOOT_TYPE}" == "warm-reboot" ]]; then + SERVICE_TO_STOP="nat radv bgp sflow lldp teamd swss syncd" + else + error "Unexpected reboot type ${REBOOT_TYPE}" + exit $EXIT_FAILURE + fi fi -debug "Stopping swss service ..." -systemctl stop swss -debug "Stopped swss service ..." +for service in ${SERVICE_TO_STOP}; do + debug "Stopping ${service} ..." -if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then - # Pre-shutdown syncd - initialize_pre_shutdown + # TODO: These exceptions for nat, sflow, lldp + # have to be coded in corresponding service scripts - if [[ "x$sonic_asic_type" == x"mellanox" ]]; then - check_issu_bank_file + if [[ "${service}" = "nat" ]]; then + /usr/local/bin/dump_nat_entries.py fi - request_pre_shutdown - - wait_for_pre_shutdown_complete_or_fail - - if [[ "x$sonic_asic_type" == x"mellanox" ]]; then - check_issu_bank_file + if [[ "${service}" = "nat" || "${service}" = "sflow" || "${service}" = "lldp" ]]; then + container kill "${service}" &> /dev/null || debug "Docker ${service} is not running ($?) ..." fi - # Warm reboot: dump state to host disk - if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then - sonic-db-cli ASIC_DB FLUSHDB > /dev/null - sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null - sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null + if [[ "${service}" = "syncd" ]]; then + systemctl stop ${service} || debug "Ignore stopping ${service} service error $?" + else + systemctl stop ${service} fi - # TODO: backup_database preserves FDB_TABLE - # need to cleanup as well for fastfast boot case - backup_database + debug "Stopped ${service}" - # Stop teamd gracefully - debug "Stopping teamd ..." - systemctl stop teamd - debug "Stopped teamd ..." -fi + if [[ "${service}" = "swss" ]]; then + if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then + # Pre-shutdown syncd + initialize_pre_shutdown + + if [[ "x$sonic_asic_type" == x"mellanox" ]]; then + check_issu_bank_file + fi -debug "Stopping syncd ..." -systemctl stop syncd || debug "Ignore stopping syncd service error $?" -debug "Stopped syncd ..." + request_pre_shutdown + + wait_for_pre_shutdown_complete_or_fail + + if [[ "x$sonic_asic_type" == x"mellanox" ]]; then + check_issu_bank_file + fi + + # Warm reboot: dump state to host disk + if [[ "$REBOOT_TYPE" = "fastfast-reboot" ]]; then + sonic-db-cli ASIC_DB FLUSHDB > /dev/null + sonic-db-cli COUNTERS_DB FLUSHDB > /dev/null + sonic-db-cli FLEX_COUNTER_DB FLUSHDB > /dev/null + fi + + # TODO: backup_database preserves FDB_TABLE + # need to cleanup as well for fastfast boot case + backup_database + fi + fi +done # Kill other containers to make the reboot faster # We call `docker kill ...` to ensure the container stops as quickly as possible, diff --git a/scripts/generate_shutdown_order.py b/scripts/generate_shutdown_order.py new file mode 100644 index 0000000000..a2427b3691 --- /dev/null +++ b/scripts/generate_shutdown_order.py @@ -0,0 +1,15 @@ +#!/usr/bin/python3 + +''' This script is used to generate initial warm/fast shutdown order file ''' + +from sonic_package_manager import PackageManager + +def main(): + manager = PackageManager.get_manager() + installed_packages = manager.get_installed_packages().values() + print('installed packages {}'.format(installed_packages)) + manager.service_creator.generate_shutdown_sequence_files(installed_packages) + print('Done.') + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 9847435671..c93ff6d48d 100644 --- a/setup.py +++ b/setup.py @@ -90,6 +90,7 @@ 'scripts/fdbshow', 'scripts/gearboxutil', 'scripts/generate_dump', + 'scripts/generate_shutdown_order.py', 'scripts/intfutil', 'scripts/intfstat', 'scripts/ipintutil', @@ -178,6 +179,7 @@ 'sonic-yang-mgmt', 'swsssdk>=2.0.1', 'tabulate==0.8.2', + 'toposort==1.6', 'www-authenticate==0.9.2', 'xmltodict==0.12.0', ], diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index f1643a1846..3c128c1a36 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -344,8 +344,13 @@ def install_from_source(self, source.install(package) exit_stack.callback(rollback_wrapper(source.uninstall, package)) - self.service_creator.create(package, state=feature_state, owner=default_owner) - exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) + self.service_creator.create(package, + installed_packages, + state=feature_state, + owner=default_owner) + exit_stack.callback(rollback_wrapper(self.service_creator.remove, + package, + installed_packages)) if not skip_cli_plugin_installation: self._install_cli_plugins(package) @@ -399,7 +404,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package) + self.service_creator.remove(package, installed_packages) # Clean containers based on this image containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) @@ -509,9 +514,13 @@ def upgrade_from_source(self, exit_stack.callback(rollback_wrapper(self._systemctl_action, old_package, 'start')) - self.service_creator.remove(old_package, deregister_feature=False) + self.service_creator.remove(old_package, + installed_packages, + deregister_feature=False) exit_stack.callback(rollback_wrapper(self.service_creator.create, - old_package, register_feature=False)) + old_package, + installed_packages, + register_feature=False)) # This is no return point, after we start removing old Docker images # there is no guaranty we can actually successfully roll-back. @@ -523,7 +532,9 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) - self.service_creator.create(new_package, register_feature=False) + self.service_creator.create(new_package, + installed_packages, + register_feature=False) if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 74da06a956..5a55c87e98 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -171,6 +171,14 @@ def unmarshal(self, value): ManifestField('asic-service', DefaultMarshaller(bool), False), ManifestField('host-service', DefaultMarshaller(bool), True), ManifestField('delayed', DefaultMarshaller(bool), False), + ManifestRoot('warm-shutdown', [ + ManifestArray('after', DefaultMarshaller(str)), + ManifestArray('before', DefaultMarshaller(str)), + ]), + ManifestRoot('fast-shutdown', [ + ManifestArray('after', DefaultMarshaller(str)), + ManifestArray('before', DefaultMarshaller(str)), + ]), ]), ManifestRoot('container', [ ManifestField('privileged', DefaultMarshaller(bool), False), @@ -184,9 +192,10 @@ def unmarshal(self, value): ManifestArray('tmpfs', DefaultMarshaller(str)), ]), ManifestArray('processes', ManifestRoot('processes', [ - ManifestField('critical', DefaultMarshaller(bool)), + ManifestField('critical', DefaultMarshaller(bool), False), ManifestField('name', DefaultMarshaller(str)), - ManifestField('command', DefaultMarshaller(str)), + ManifestField('command', DefaultMarshaller(str), ''), + ManifestField('reconciles', DefaultMarshaller(bool), False), ])), ManifestRoot('cli', [ ManifestField('mandatory', DefaultMarshaller(bool), False), diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index f62d0a3074..978d53bf0d 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -3,16 +3,17 @@ import os import stat import subprocess -from typing import Dict +from collections import defaultdict +from typing import Dict, List import jinja2 as jinja2 from prettyprinter import pformat - from sonic_package_manager.logger import log from sonic_package_manager.package import Package from sonic_package_manager.service_creator import ETC_SONIC_PATH from sonic_package_manager.service_creator.feature import FeatureRegistry from sonic_package_manager.service_creator.utils import in_chroot +from toposort import toposort_flatten, CircularDependencyError SERVICE_FILE_TEMPLATE = 'sonic.service.j2' TIMER_UNIT_TEMPLATE = 'timer.unit.j2' @@ -108,9 +109,11 @@ def __init__(self, feature_registry: FeatureRegistry, sonic_db): def create(self, package: Package, + all_packages: List[Package] = None, register_feature=True, state='enabled', owner='local'): + all_packages = all_packages or [] try: self.generate_container_mgmt(package) self.generate_service_mgmt(package) @@ -118,10 +121,11 @@ def create(self, self.generate_systemd_service(package) self.generate_monit_conf(package) self.generate_dump_script(package) + self.generate_service_reconciliation_file(package) self.set_initial_config(package) - self.post_operation_hook() + self.post_operation_hook(all_packages) if register_feature: self.feature_registry.register(package.manifest, @@ -130,7 +134,11 @@ def create(self, self.remove(package, not register_feature) raise - def remove(self, package: Package, deregister_feature=True): + def remove(self, + package: Package, + all_packages: List[Package] = None, + deregister_feature=True): + all_packages = all_packages or [] name = package.manifest['service']['name'] def remove_file(path): @@ -144,18 +152,21 @@ def remove_file(path): remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) remove_file(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) remove_file(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) + remove_file(os.path.join(ETC_SONIC_PATH, f'{name}_reconcile')) self.update_dependent_list_file(package, remove=True) - self.post_operation_hook() + self.post_operation_hook(all_packages) if deregister_feature: self.feature_registry.deregister(package.manifest['service']['name']) - def post_operation_hook(self): + def post_operation_hook(self, all_packages: List[Package]): if not in_chroot(): run_command('systemctl daemon-reload') run_command('systemctl reload monit') + + self.generate_shutdown_sequence_files(all_packages) def generate_container_mgmt(self, package: Package): image_id = package.image_id @@ -303,6 +314,46 @@ def generate_dump_script(self, package): render_template(scrip_template, script_path, render_ctx, executable=True) log.info(f'generated {script_path}') + def generate_shutdown_sequence(self, installed_packages, reboot_type): + shutdown_graph = dict() + for package in installed_packages: + after = set(package.manifest['service'][f'{reboot_type}-shutdown']['after']) + before = set(package.manifest['service'][f'{reboot_type}-shutdown']['before']) + if not after and not before: + continue + shutdown_graph.setdefault(package.name, set()) + shutdown_graph[package.name].update(after) + + for service in before: + shutdown_graph.setdefault(service, set()) + shutdown_graph[service].update({package.name}) + + log.debug(f'shutdown graph {pformat(shutdown_graph)}') + + try: + order = toposort_flatten(shutdown_graph) + except CircularDependencyError as err: + raise ServiceCreatorError(f'Circular dependency found in {reboot_type} shutdown graph: {err}') + + log.debug(f'shutdown order {pformat(order)}') + return order + + def generate_shutdown_sequence_file(self, installed_packages, reboot_type): + order = self.generate_shutdown_sequence(installed_packages, reboot_type) + with open(os.path.join(ETC_SONIC_PATH, f'{reboot_type}-reboot_order'), 'w') as file: + file.write(' '.join(order)) + + def generate_shutdown_sequence_files(self, installed_packages): + for reboot_type in ('fast', 'warm'): + self.generate_shutdown_sequence_file(installed_packages, reboot_type) + + def generate_service_reconciliation_file(self, package): + name = package.manifest['service']['name'] + processes = [process['name'] for process in package.manifest['processes'] + if process['reconciles']] + with open(os.path.join(ETC_SONIC_PATH, f'{name}_reconcile'), 'w') as file: + file.write(' '.join(processes)) + def set_initial_config(self, package): init_cfg = package.manifest['package']['init-cfg'] diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index bf787bdab4..fc8005212d 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -75,6 +75,35 @@ def __init__(self): components={ 'libswsscommon': Version.parse('1.0.0'), 'libsairedis': Version.parse('1.0.0') + }, + warm_shutdown={ + 'before': ['syncd'], + }, + fast_shutdown={ + 'before': ['syncd'], + }, + processes=[ + { + 'name': 'orchagent', + 'reconciles': True, + }, + { + 'name': 'neighsyncd', + 'reconciles': True, + } + ], + ) + self.add('docker-teamd', 'latest', 'teamd', '1.0.0', + components={ + 'libswsscommon': Version.parse('1.0.0'), + 'libsairedis': Version.parse('1.0.0') + }, + warm_shutdown={ + 'before': ['syncd'], + 'after': ['swss'], + }, + fast_shutdown={ + 'before': ['swss'], } ) self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') @@ -108,7 +137,9 @@ def from_tarball(self, filepath: str) -> Manifest: components = self.metadata_store[path][ref]['components'] return Metadata(manifest, components) - def add(self, repo, reference, name, version, components=None): + def add(self, repo, reference, name, version, components=None, + warm_shutdown=None, fast_shutdown=None, + processes=None): repo_dict = self.metadata_store.setdefault(repo, {}) repo_dict[reference] = { 'manifest': { @@ -119,7 +150,10 @@ def add(self, repo, reference, name, version, components=None): }, 'service': { 'name': name, - } + 'warm-shutdown': warm_shutdown or {}, + 'fast-shutdown': fast_shutdown or {}, + }, + 'processes': processes or {} }, 'components': components or {}, } @@ -189,6 +223,16 @@ def fake_db(fake_metadata_resolver): installed=True, built_in=True ) + add_package( + content, + fake_metadata_resolver, + 'docker-teamd', + 'latest', + description='SONiC teamd service', + default_reference='1.0.0', + installed=True, + built_in=True + ) add_package( content, fake_metadata_resolver, diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index 256832118e..ee0b817624 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -250,12 +250,12 @@ def test_upgrade_from_file_known_package(package_manager, fake_db, sonic_fs): def test_installation_non_default_owner(package_manager, anything, mock_service_creator): package_manager.install('test-package', default_owner='kube') - mock_service_creator.create.assert_called_once_with(anything, state='disabled', owner='kube') + mock_service_creator.create.assert_called_once_with(anything, anything, state='disabled', owner='kube') def test_installation_enabled(package_manager, anything, mock_service_creator): package_manager.install('test-package', enable=True) - mock_service_creator.create.assert_called_once_with(anything, state='enabled', owner='local') + mock_service_creator.create.assert_called_once_with(anything, anything, state='enabled', owner='local') def test_installation_fault(package_manager, mock_docker_api, mock_service_creator): diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index e69b4447f8..006b1af4ea 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -37,15 +37,30 @@ def manifest(): 'volumes': [ '/etc/sonic:/etc/sonic:ro' ] - } + }, + 'processes': [ + { + 'name': 'test-process', + 'reconciles': True, + }, + { + 'name': 'test-process-2', + 'reconciles': False, + }, + { + 'name': 'test-process-3', + 'reconciles': True, + }, + ] }) -def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): +def test_service_creator(sonic_fs, manifest, package_manager, mock_feature_registry, mock_sonic_db): creator = ServiceCreator(mock_feature_registry, mock_sonic_db) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + installed_packages = package_manager.get_installed_packages().values() + creator.create(package, all_packages=installed_packages) assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) @@ -53,6 +68,14 @@ def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_d assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) assert sonic_fs.exists(os.path.join(MONIT_CONF_LOCATION, 'monit_test')) + def read_file(name): + with open(os.path.join(ETC_SONIC_PATH, name)) as file: + return file.read() + + assert read_file('warm-reboot_order') == 'swss teamd syncd' + assert read_file('fast-reboot_order') == 'teamd swss syncd' + assert read_file('test_reconcile') == 'test-process test-process-3' + def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): creator = ServiceCreator(mock_feature_registry, mock_sonic_db) From c7245694eef3c33a12c71ef137a33c2cf90e284c Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 15:14:51 +0200 Subject: [PATCH 13/60] iteritems -> items Signed-off-by: Stepan Blyschak --- sonic_package_manager/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py index e5f9dbb3a5..99246f1c6a 100644 --- a/sonic_package_manager/metadata.py +++ b/sonic_package_manager/metadata.py @@ -22,7 +22,7 @@ def deep_update(dst: Dict, src: Dict) -> Dict: New merged dictionary. """ - for key, value in src.iteritems(): + for key, value in src.items(): if isinstance(value, dict): node = dst.setdefault(key, {}) deep_update(node, value) From 3dc793fa8f81ead54c37b183dc5ac76c3618a0a2 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 16:06:58 +0200 Subject: [PATCH 14/60] fix issue in metadata dict generation Signed-off-by: Stepan Blyschak --- sonic_package_manager/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/metadata.py b/sonic_package_manager/metadata.py index 99246f1c6a..7f7c25ceaf 100644 --- a/sonic_package_manager/metadata.py +++ b/sonic_package_manager/metadata.py @@ -61,8 +61,8 @@ def translate_plain_to_tree(plain: Dict[str, str], sep='.') -> Dict: res[key] = value continue namespace, key = key.split(sep, 1) - res.setdefault(key, {}) - deep_update(res[key], translate_plain_to_tree({key: value})) + res.setdefault(namespace, {}) + deep_update(res[namespace], translate_plain_to_tree({key: value})) return res From ea148672609fc719a6fe0388741e07bf9f44249a Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 17:16:37 +0200 Subject: [PATCH 15/60] fix creator with service name Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 978d53bf0d..1f3a15c80d 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -321,12 +321,13 @@ def generate_shutdown_sequence(self, installed_packages, reboot_type): before = set(package.manifest['service'][f'{reboot_type}-shutdown']['before']) if not after and not before: continue + name = package.manifest['service']['name'] shutdown_graph.setdefault(package.name, set()) - shutdown_graph[package.name].update(after) + shutdown_graph[name].update(after) for service in before: shutdown_graph.setdefault(service, set()) - shutdown_graph[service].update({package.name}) + shutdown_graph[service].update({name}) log.debug(f'shutdown graph {pformat(shutdown_graph)}') From faeadcf12b1829d24c9762b01f1d87b320d1909c Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 25 Mar 2021 17:20:06 +0200 Subject: [PATCH 16/60] default value for component constraint Signed-off-by: Stepan Blyschak --- sonic_package_manager/manifest.py | 2 +- tests/sonic_package_manager/test_manifest.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 74da06a956..5a357d668d 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -150,7 +150,7 @@ def unmarshal(self, value): ManifestField('version', ParsedMarshaller(Version)), ManifestField('name', DefaultMarshaller(str)), ManifestField('description', DefaultMarshaller(str), ''), - ManifestField('base-os', ParsedMarshaller(ComponentConstraints), dict()), + ManifestField('base-os', ParsedMarshaller(ComponentConstraints), ComponentConstraints()), ManifestArray('depends', ParsedMarshaller(PackageConstraint)), ManifestArray('breaks', ParsedMarshaller(PackageConstraint)), ManifestField('init-cfg', DefaultMarshaller(dict), dict()), diff --git a/tests/sonic_package_manager/test_manifest.py b/tests/sonic_package_manager/test_manifest.py index f6028e0437..cf3c02b6d1 100644 --- a/tests/sonic_package_manager/test_manifest.py +++ b/tests/sonic_package_manager/test_manifest.py @@ -2,6 +2,7 @@ import pytest +from sonic_package_manager.constraint import ComponentConstraints from sonic_package_manager.manifest import Manifest, ManifestError from sonic_package_manager.version import VersionRange @@ -12,7 +13,7 @@ def test_manifest_v1_defaults(): 'service': {'name': 'test'}}) assert manifest['package']['depends'] == [] assert manifest['package']['breaks'] == [] - assert manifest['package']['base-os'] == dict() + assert manifest['package']['base-os'] == ComponentConstraints() assert not manifest['service']['asic-service'] assert manifest['service']['host-service'] From 31138d051721216d3a8d0ae6cae1f0dbb5803019 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 26 Mar 2021 11:24:29 +0200 Subject: [PATCH 17/60] fix passing installed_packages Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 12 ++++++------ sonic_package_manager/service_creator/creator.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 3c128c1a36..75390f30cd 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -345,12 +345,12 @@ def install_from_source(self, exit_stack.callback(rollback_wrapper(source.uninstall, package)) self.service_creator.create(package, - installed_packages, + installed_packages.values(), state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package, - installed_packages)) + installed_packages.values())) if not skip_cli_plugin_installation: self._install_cli_plugins(package) @@ -404,7 +404,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package, installed_packages) + self.service_creator.remove(package, installed_packages.values()) # Clean containers based on this image containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) @@ -515,11 +515,11 @@ def upgrade_from_source(self, old_package, 'start')) self.service_creator.remove(old_package, - installed_packages, + installed_packages.values(), deregister_feature=False) exit_stack.callback(rollback_wrapper(self.service_creator.create, old_package, - installed_packages, + installed_packages.values(), register_feature=False)) # This is no return point, after we start removing old Docker images @@ -533,7 +533,7 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) self.service_creator.create(new_package, - installed_packages, + installed_packages.values(), register_feature=False) if self.feature_registry.is_feature_enabled(new_feature): diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 1f3a15c80d..6299ed3747 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -322,7 +322,7 @@ def generate_shutdown_sequence(self, installed_packages, reboot_type): if not after and not before: continue name = package.manifest['service']['name'] - shutdown_graph.setdefault(package.name, set()) + shutdown_graph.setdefault(name, set()) shutdown_graph[name].update(after) for service in before: From ae1ce45604d69ad24627754728a898236447c733 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Mon, 29 Mar 2021 10:20:57 +0300 Subject: [PATCH 18/60] Update doc/Command-Reference.md Co-authored-by: Joe LeVeque --- doc/Command-Reference.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index b55e36d53d..36837ee4c2 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7962,7 +7962,9 @@ Go Back To [Beginning of the document](#) or [Beginning of this section](#waterm ## Software Installation and Management -SONiC software image can be installed in two methods, viz, "using sonic-installer tool", "ONIE Installer". +SONiC image can be installed in one of two methods: +1. From within a running SONiC iamge using the `sonic-installer` utility +2. From the vendor's bootloader (E.g., ONIE, Aboot, etc.) SONiC feature Docker images (aka "SONiC packages") available to be installed with *sonic-package-manager* utility. From 5c045e0c50fbb286fe88881fc21735d59bd5dbb6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 30 Mar 2021 12:38:02 +0300 Subject: [PATCH 19/60] use defaultdict Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 6299ed3747..5aa19c6c49 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -315,18 +315,16 @@ def generate_dump_script(self, package): log.info(f'generated {script_path}') def generate_shutdown_sequence(self, installed_packages, reboot_type): - shutdown_graph = dict() + shutdown_graph = defaultdict(set) for package in installed_packages: after = set(package.manifest['service'][f'{reboot_type}-shutdown']['after']) before = set(package.manifest['service'][f'{reboot_type}-shutdown']['before']) if not after and not before: continue name = package.manifest['service']['name'] - shutdown_graph.setdefault(name, set()) shutdown_graph[name].update(after) for service in before: - shutdown_graph.setdefault(service, set()) shutdown_graph[service].update({name}) log.debug(f'shutdown graph {pformat(shutdown_graph)}') From a1a97f958e744fdf882f942a096110d881711a36 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 30 Mar 2021 18:03:32 +0300 Subject: [PATCH 20/60] fix internal review comments Signed-off-by: Stepan Blyschak --- scripts/fast-reboot | 8 +- sonic_package_manager/manager.py | 34 ++++++--- .../service_creator/creator.py | 75 ++++++++++++++----- tests/sonic_package_manager/conftest.py | 11 +++ .../test_service_creator.py | 27 ++++--- 5 files changed, 114 insertions(+), 41 deletions(-) diff --git a/scripts/fast-reboot b/scripts/fast-reboot index d11b143d56..e67fded528 100755 --- a/scripts/fast-reboot +++ b/scripts/fast-reboot @@ -549,20 +549,20 @@ fi if [[ -f ${SHUTDOWN_ORDER_FILE} ]]; then - SERVICE_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})" + SERVICES_TO_STOP="$(cat ${SHUTDOWN_ORDER_FILE})" else # TODO: to be removed once sonic-buildimage change is in if [[ "${REBOOT_TYPE}" == "fast-reboot" ]]; then - SERVICE_TO_STOP="nat radv bgp sflow lldp swss teamd syncd" + SERVICES_TO_STOP="nat radv bgp sflow lldp swss teamd syncd" elif [[ "${REBOOT_TYPE}" == "fastfast-reboot" || "${REBOOT_TYPE}" == "warm-reboot" ]]; then - SERVICE_TO_STOP="nat radv bgp sflow lldp teamd swss syncd" + SERVICES_TO_STOP="nat radv bgp sflow lldp teamd swss syncd" else error "Unexpected reboot type ${REBOOT_TYPE}" exit $EXIT_FAILURE fi fi -for service in ${SERVICE_TO_STOP}; do +for service in ${SERVICES_TO_STOP}; do debug "Stopping ${service} ..." # TODO: These exceptions for nat, sflow, lldp diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 75390f30cd..660da49093 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -4,7 +4,14 @@ import os import pkgutil import tempfile -from typing import Any, Iterable, Callable, Dict, Optional +from typing import ( + Any, + Iterable, + List, + Callable, + Dict, + Optional, +) import docker import filelock @@ -345,12 +352,12 @@ def install_from_source(self, exit_stack.callback(rollback_wrapper(source.uninstall, package)) self.service_creator.create(package, - installed_packages.values(), + self.get_installed_packages().values(), state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package, - installed_packages.values())) + self.get_installed_packages().values())) if not skip_cli_plugin_installation: self._install_cli_plugins(package) @@ -404,7 +411,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package, installed_packages.values()) + self.service_creator.remove(package, self.get_installed_packages().values()) # Clean containers based on this image containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) @@ -515,11 +522,11 @@ def upgrade_from_source(self, old_package, 'start')) self.service_creator.remove(old_package, - installed_packages.values(), + self.get_installed_packages().values(), deregister_feature=False) exit_stack.callback(rollback_wrapper(self.service_creator.create, old_package, - installed_packages.values(), + self.get_installed_packages().values(), register_feature=False)) # This is no return point, after we start removing old Docker images @@ -533,7 +540,7 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) self.service_creator.create(new_package, - installed_packages.values(), + self.get_installed_packages().values(), register_feature=False) if self.feature_registry.is_feature_enabled(new_feature): @@ -793,10 +800,19 @@ def get_installed_packages(self) -> Dict[str, Package]: """ return { - entry.name: self.get_installed_package(entry.name) - for entry in self.database if entry.installed + entry.name: entry for entry in self.get_installed_packages_list() } + def get_installed_packages_list(self) -> List[Package]: + """ Returns a list of installed packages. + + Returns: + Installed packages dictionary. + """ + + return [self.get_installed_package(entry.name) + for entry in self.database if entry.installed] + def _migrate_package_database(self, old_package_database: PackageDatabase): """ Performs part of package migration process. For every package in old_package_database that is not listed in current diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 5aa19c6c49..a7b95cbf69 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -94,7 +94,7 @@ def run_command(command: str): shell=True, executable='/bin/bash', stdout=subprocess.PIPE) - (out, _) = proc.communicate() + (_, _) = proc.communicate() if proc.returncode != 0: raise ServiceCreatorError(f'Failed to execute "{command}"') @@ -109,11 +109,23 @@ def __init__(self, feature_registry: FeatureRegistry, sonic_db): def create(self, package: Package, - all_packages: List[Package] = None, - register_feature=True, - state='enabled', - owner='local'): - all_packages = all_packages or [] + all_packages: List[Package], + register_feature: bool = True, + state: str = 'enabled', + owner: str = 'local'): + """ Register package as SONiC service. + + Args: + package: Package object to install. + all_packages: List of installed packages. + register_feature: Wether to register this package in FEATURE table. + state: Default feature state. + owner: Default feature owner. + + Returns: + None + """ + try: self.generate_container_mgmt(package) self.generate_service_mgmt(package) @@ -125,20 +137,31 @@ def create(self, self.set_initial_config(package) - self.post_operation_hook(all_packages) + self.post_operation_hook(all_packages + [package]) if register_feature: self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): - self.remove(package, not register_feature) + self.remove(package, all_packages + [package], + deregister_feature=not register_feature) raise def remove(self, package: Package, - all_packages: List[Package] = None, - deregister_feature=True): - all_packages = all_packages or [] + all_packages: List[Package], + deregister_feature: bool = True): + """ Uninstall SONiC service provided by the package. + + Args: + package: Package object to uninstall. + all_packages: List of installed packages. + deregister_feature: Wether to deregister this package from FEATURE table. + + Returns: + None + """ + name = package.manifest['service']['name'] def remove_file(path): @@ -156,6 +179,8 @@ def remove_file(path): self.update_dependent_list_file(package, remove=True) + # remove package that is going to be uninstalled from installed list + all_packages.remove(package) self.post_operation_hook(all_packages) if deregister_feature: @@ -314,13 +339,27 @@ def generate_dump_script(self, package): render_template(scrip_template, script_path, render_ctx, executable=True) log.info(f'generated {script_path}') - def generate_shutdown_sequence(self, installed_packages, reboot_type): + def generate_shutdown_sequence(self, all_packages, reboot_type): shutdown_graph = defaultdict(set) - for package in installed_packages: - after = set(package.manifest['service'][f'{reboot_type}-shutdown']['after']) - before = set(package.manifest['service'][f'{reboot_type}-shutdown']['before']) + + def service_exists(service): + for package in all_packages: + if package.manifest['service']['name'] == service: + return True + log.info(f'Service {service} is not installed, it is skipped...') + return False + + def filter_not_available(services): + return set(filter(service_exists, services)) + + for package in all_packages: + service_props = package.manifest['service'] + after = filter_not_available(service_props[f'{reboot_type}-shutdown']['after']) + before = filter_not_available(service_props[f'{reboot_type}-shutdown']['before']) + if not after and not before: continue + name = package.manifest['service']['name'] shutdown_graph[name].update(after) @@ -332,7 +371,7 @@ def generate_shutdown_sequence(self, installed_packages, reboot_type): try: order = toposort_flatten(shutdown_graph) except CircularDependencyError as err: - raise ServiceCreatorError(f'Circular dependency found in {reboot_type} shutdown graph: {err}') + raise ServiceCreatorError(f'Circular dependency found in {reboot_type} error: {err}') log.debug(f'shutdown order {pformat(order)}') return order @@ -348,8 +387,8 @@ def generate_shutdown_sequence_files(self, installed_packages): def generate_service_reconciliation_file(self, package): name = package.manifest['service']['name'] - processes = [process['name'] for process in package.manifest['processes'] - if process['reconciles']] + all_processes = package.manifest['processes'] + processes = [process['name'] for process in all_processes if process['reconciles']] with open(os.path.join(ETC_SONIC_PATH, f'{name}_reconcile'), 'w') as file: file.write(' '.join(processes)) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index fc8005212d..d7a9db3e57 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -93,6 +93,7 @@ def __init__(self): } ], ) + self.add('docker-syncd', 'latest', 'syncd', '1.0.0') self.add('docker-teamd', 'latest', 'teamd', '1.0.0', components={ 'libswsscommon': Version.parse('1.0.0'), @@ -223,6 +224,16 @@ def fake_db(fake_metadata_resolver): installed=True, built_in=True ) + add_package( + content, + fake_metadata_resolver, + 'docker-syncd', + 'latest', + description='SONiC syncd service', + default_reference='1.0.0', + installed=True, + built_in=True + ) add_package( content, fake_metadata_resolver, diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index 006b1af4ea..37c83ed16a 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -31,6 +31,13 @@ def manifest(): 'dependent-of': ['swss'], 'asic-service': False, 'host-service': True, + 'warm-shutdown': { + 'before': ['syncd'], + 'after': ['swss'], + }, + 'fast-shutdown': { + 'before': ['swss'], + }, }, 'container': { 'privileged': True, @@ -59,8 +66,8 @@ def test_service_creator(sonic_fs, manifest, package_manager, mock_feature_regis creator = ServiceCreator(mock_feature_registry, mock_sonic_db) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - installed_packages = package_manager.get_installed_packages().values() - creator.create(package, all_packages=installed_packages) + installed_packages = list(package_manager.get_installed_packages().values()) + [package] + creator.create(package, installed_packages) assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) @@ -72,8 +79,8 @@ def read_file(name): with open(os.path.join(ETC_SONIC_PATH, name)) as file: return file.read() - assert read_file('warm-reboot_order') == 'swss teamd syncd' - assert read_file('fast-reboot_order') == 'teamd swss syncd' + assert read_file('warm-reboot_order') == 'swss teamd test syncd' + assert read_file('fast-reboot_order') == 'teamd test swss syncd' assert read_file('test_reconcile') == 'test-process test-process-3' @@ -81,13 +88,13 @@ def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_regist creator = ServiceCreator(mock_feature_registry, mock_sonic_db) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert not sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) manifest['service']['delayed'] = True package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) @@ -96,13 +103,13 @@ def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_regist creator = ServiceCreator(mock_feature_registry, mock_sonic_db) entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) manifest['package']['debug-dump'] = '/some/command' package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) @@ -118,7 +125,7 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr entry = PackageEntry('test', 'azure/sonic-test') package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) @@ -132,7 +139,7 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr } package = Package(entry, Metadata(manifest)) - creator.create(package) + creator.create(package, []) mock_table.set.assert_called_with('key_a', [('field_1', 'value_1'), ('field_2', 'original_value_2')]) From 7a6c8d145301668b3b4e0b11173dade618faca8d Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 30 Mar 2021 18:10:13 +0300 Subject: [PATCH 21/60] fix internal review comments Signed-off-by: Stepan Blyschak --- scripts/generate_shutdown_order.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_shutdown_order.py b/scripts/generate_shutdown_order.py index a2427b3691..db9f48d676 100644 --- a/scripts/generate_shutdown_order.py +++ b/scripts/generate_shutdown_order.py @@ -6,7 +6,7 @@ def main(): manager = PackageManager.get_manager() - installed_packages = manager.get_installed_packages().values() + installed_packages = manager.get_installed_packages_list() print('installed packages {}'.format(installed_packages)) manager.service_creator.generate_shutdown_sequence_files(installed_packages) print('Done.') From c4b4225d1199bdfbf766b51e6310ed349f4f7efe Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 31 Mar 2021 14:57:08 +0300 Subject: [PATCH 22/60] use get_installed_packages_list() function Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 660da49093..53d632a59a 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -352,12 +352,12 @@ def install_from_source(self, exit_stack.callback(rollback_wrapper(source.uninstall, package)) self.service_creator.create(package, - self.get_installed_packages().values(), + self.get_installed_packages_list(), state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package, - self.get_installed_packages().values())) + self.get_installed_packages_list())) if not skip_cli_plugin_installation: self._install_cli_plugins(package) @@ -411,7 +411,7 @@ def uninstall(self, name: str, force=False): try: self._uninstall_cli_plugins(package) - self.service_creator.remove(package, self.get_installed_packages().values()) + self.service_creator.remove(package, self.get_installed_packages_list()) # Clean containers based on this image containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) @@ -522,11 +522,11 @@ def upgrade_from_source(self, old_package, 'start')) self.service_creator.remove(old_package, - self.get_installed_packages().values(), + self.get_installed_packages_list(), deregister_feature=False) exit_stack.callback(rollback_wrapper(self.service_creator.create, old_package, - self.get_installed_packages().values(), + self.get_installed_packages_list(), register_feature=False)) # This is no return point, after we start removing old Docker images @@ -540,7 +540,7 @@ def upgrade_from_source(self, self.docker.rmi(old_package.image_id, force=True) self.service_creator.create(new_package, - self.get_installed_packages().values(), + self.get_installed_packages_list(), register_feature=False) if self.feature_registry.is_feature_enabled(new_feature): From f32d038de8ef6ed49a54b6d5cc09549248bfe7f5 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 31 Mar 2021 18:11:00 +0300 Subject: [PATCH 23/60] dont fail uninstall when no package in installed list Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index a7b95cbf69..6622720168 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -143,7 +143,7 @@ def create(self, self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): - self.remove(package, all_packages + [package], + self.remove(package, all_packages, deregister_feature=not register_feature) raise @@ -179,8 +179,8 @@ def remove_file(path): self.update_dependent_list_file(package, remove=True) - # remove package that is going to be uninstalled from installed list - all_packages.remove(package) + # make sure package that is going to be uninstalled is not in installed list. + with contextlib.suppress(ValueError): all_packages.remove(package) self.post_operation_hook(all_packages) if deregister_feature: From 83f2af68a1d4dec87649b7918c4ed7ec2206fd73 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 1 Apr 2021 12:40:53 +0300 Subject: [PATCH 24/60] drop support for monit as monit is getting deprecated Signed-off-by: Stepan Blyschak --- sonic-utilities-data/templates/monit.conf.j2 | 18 ------------------ sonic_package_manager/constraint.py | 1 - sonic_package_manager/manifest.py | 2 -- .../service_creator/creator.py | 16 ---------------- tests/sonic_package_manager/conftest.py | 2 -- .../test_service_creator.py | 7 +------ 6 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 sonic-utilities-data/templates/monit.conf.j2 diff --git a/sonic-utilities-data/templates/monit.conf.j2 b/sonic-utilities-data/templates/monit.conf.j2 deleted file mode 100644 index f51efb9bee..0000000000 --- a/sonic-utilities-data/templates/monit.conf.j2 +++ /dev/null @@ -1,18 +0,0 @@ -############################################################################### -## -## =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== -## auto-generated from {{ source }} by sonic-package-manager -## -## Monit configuration for {{ feature }} service -## process list: -{%- for process in processes %} -{%- if process.critical %} -## {{ process.name }} -{%- endif %} -{%- endfor %} -############################################################################### -{%- for process in processes %} -check program {{ feature }}|{{ process.name }} with path "/usr/bin/process_checker {{ feature }} {{ process.command }}" - if status != 0 for 5 times within 5 cycles then alert - -{% endfor %} diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py index 09f0fbc0fe..7dde13377f 100644 --- a/sonic_package_manager/constraint.py +++ b/sonic_package_manager/constraint.py @@ -137,4 +137,3 @@ def parse(constraint: Union[str, Dict]) -> 'PackageConstraint': return PackageConstraint.from_dict(constraint) else: raise ValueError('Input argument should be either str or dict') - diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 5a357d668d..371ed145ba 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -184,9 +184,7 @@ def unmarshal(self, value): ManifestArray('tmpfs', DefaultMarshaller(str)), ]), ManifestArray('processes', ManifestRoot('processes', [ - ManifestField('critical', DefaultMarshaller(bool)), ManifestField('name', DefaultMarshaller(str)), - ManifestField('command', DefaultMarshaller(str)), ])), ManifestRoot('cli', [ ManifestField('mandatory', DefaultMarshaller(bool), False), diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index f62d0a3074..53fa51075d 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -25,9 +25,6 @@ DOCKER_CTL_SCRIPT_TEMPLATE = 'docker_image_ctl.j2' DOCKER_CTL_SCRIPT_LOCATION = '/usr/bin' -MONIT_CONF_TEMPLATE = 'monit.conf.j2' -MONIT_CONF_LOCATION = '/etc/monit/conf.d/' - DEBUG_DUMP_SCRIPT_TEMPLATE = 'dump.sh.j2' DEBUG_DUMP_SCRIPT_LOCATION = '/usr/local/bin/debug-dump/' @@ -116,7 +113,6 @@ def create(self, self.generate_service_mgmt(package) self.update_dependent_list_file(package) self.generate_systemd_service(package) - self.generate_monit_conf(package) self.generate_dump_script(package) self.set_initial_config(package) @@ -138,7 +134,6 @@ def remove_file(path): os.remove(path) log.info(f'removed {path}') - remove_file(os.path.join(MONIT_CONF_LOCATION, f'monit_{name}')) remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}.service')) remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}@.service')) remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) @@ -155,7 +150,6 @@ def remove_file(path): def post_operation_hook(self): if not in_chroot(): run_command('systemctl daemon-reload') - run_command('systemctl reload monit') def generate_container_mgmt(self, package: Package): image_id = package.image_id @@ -243,16 +237,6 @@ def generate_systemd_service(self, package: Package): render_template(template, output_file, template_vars) log.info(f'generated {output_file}') - def generate_monit_conf(self, package: Package): - name = package.manifest['service']['name'] - processes = package.manifest['processes'] - output_filename = os.path.join(MONIT_CONF_LOCATION, f'monit_{name}') - render_template(get_tmpl_path(MONIT_CONF_TEMPLATE), output_filename, - {'source': get_tmpl_path(MONIT_CONF_TEMPLATE), - 'feature': name, - 'processes': processes}) - log.info(f'generated {output_filename}') - def update_dependent_list_file(self, package: Package, remove=False): name = package.manifest['service']['name'] dependent_of = package.manifest['service']['dependent-of'] diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index bf787bdab4..d29f449684 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -337,12 +337,10 @@ def sonic_fs(fs): fs.create_dir(SYSTEMD_LOCATION) fs.create_dir(DOCKER_CTL_SCRIPT_LOCATION) fs.create_dir(SERVICE_MGMT_SCRIPT_LOCATION) - fs.create_dir(MONIT_CONF_LOCATION) fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_FILE_TEMPLATE)) fs.create_file(os.path.join(TEMPLATES_PATH, TIMER_UNIT_TEMPLATE)) fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_MGMT_SCRIPT_TEMPLATE)) fs.create_file(os.path.join(TEMPLATES_PATH, DOCKER_CTL_SCRIPT_TEMPLATE)) - fs.create_file(os.path.join(TEMPLATES_PATH, MONIT_CONF_TEMPLATE)) fs.create_file(os.path.join(TEMPLATES_PATH, DEBUG_DUMP_SCRIPT_TEMPLATE)) yield fs diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index e69b4447f8..f6ef317d8c 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -8,11 +8,7 @@ from sonic_package_manager.manifest import Manifest from sonic_package_manager.metadata import Metadata from sonic_package_manager.package import Package -from sonic_package_manager.service_creator.creator import ( - ServiceCreator, - ETC_SONIC_PATH, DOCKER_CTL_SCRIPT_LOCATION, - SERVICE_MGMT_SCRIPT_LOCATION, SYSTEMD_LOCATION, MONIT_CONF_LOCATION, DEBUG_DUMP_SCRIPT_LOCATION -) +from sonic_package_manager.service_creator.creator import * from sonic_package_manager.service_creator.feature import FeatureRegistry @@ -51,7 +47,6 @@ def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_d assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) assert sonic_fs.exists(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, 'test.sh')) assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) - assert sonic_fs.exists(os.path.join(MONIT_CONF_LOCATION, 'monit_test')) def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): From d31497ab6dfb76bba11564e700e5133ae5ce388f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 1 Apr 2021 16:32:53 +0300 Subject: [PATCH 25/60] fix manifest representation Signed-off-by: Stepan Blyschak --- sonic_package_manager/constraint.py | 37 +++++++++++++++++--- sonic_package_manager/manifest.py | 2 ++ tests/sonic_package_manager/test_manifest.py | 8 ++++- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py index 7dde13377f..8b044f3ec9 100644 --- a/sonic_package_manager/constraint.py +++ b/sonic_package_manager/constraint.py @@ -49,6 +49,18 @@ def parse(constraints: Dict) -> 'ComponentConstraints': for component, version in constraints.items()} return ComponentConstraints(components) + def deparse(self) -> Dict[str, str]: + """ Returns the manifest representation of components constraints. + + Returns: + Dictionary of string keys and string values. + + """ + + return { + component: str(version) for component, version in self.components + } + @dataclass class PackageConstraint: @@ -56,10 +68,12 @@ class PackageConstraint: name: str constraint: VersionConstraint - components: Dict[str, VersionConstraint] = field(default_factory=dict) + _components: ComponentConstraints = ComponentConstraints({}) + + def __str__(self): return f'{self.name}{self.constraint}' - def __str__(self): - return f'{self.name}{self.constraint}' + @property + def components(self): return self._components.components @staticmethod def from_string(constraint_expression: str) -> 'PackageConstraint': @@ -115,8 +129,7 @@ def from_dict(constraint_dict: Dict) -> 'PackageConstraint': name = constraint_dict['name'] version = VersionConstraint.parse(constraint_dict.get('version') or '*') - components = {component: VersionConstraint.parse(version) - for component, version in constraint_dict.get('components', {}).items()} + components = ComponentConstraints.parse(constraint_dict.get('components', {})) return PackageConstraint(name, version, components) @staticmethod @@ -137,3 +150,17 @@ def parse(constraint: Union[str, Dict]) -> 'PackageConstraint': return PackageConstraint.from_dict(constraint) else: raise ValueError('Input argument should be either str or dict') + + def deparse(self) -> Dict: + """ Returns the manifest representation of package constraint. + + Returns: + Dictionary in manifest representation. + + """ + + return { + 'name': self.name, + 'version': str(self.constraint), + 'components': self._components.deparse(), + } diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 371ed145ba..451b28df16 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -54,6 +54,8 @@ def marshal(self, value): def unmarshal(self, value): try: + if hasattr(value, 'deparse'): + return value.deparse() return str(value) except Exception as err: raise ManifestError(f'Failed to unmarshal {value}: {err}') diff --git a/tests/sonic_package_manager/test_manifest.py b/tests/sonic_package_manager/test_manifest.py index cf3c02b6d1..efdcc558ab 100644 --- a/tests/sonic_package_manager/test_manifest.py +++ b/tests/sonic_package_manager/test_manifest.py @@ -59,7 +59,13 @@ def test_manifest_v1_mounts_invalid(): def test_manifest_v1_unmarshal(): manifest_json_input = {'package': {'name': 'test', 'version': '1.0.0', - 'depends': ['swss>1.0.0']}, + 'depends': [ + { + 'name': 'swss', + 'version': '>1.0.0', + 'components': {}, + } + ]}, 'service': {'name': 'test'}} manifest = Manifest.marshal(manifest_json_input) manifest_json = manifest.unmarshal() From c2eacdff5265fe7595514e79354d3030b553d7b9 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 1 Apr 2021 16:47:56 +0300 Subject: [PATCH 26/60] fix LGTM warning Signed-off-by: Stepan Blyschak --- sonic_package_manager/manifest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 451b28df16..1bd5449d3a 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -5,11 +5,10 @@ from sonic_package_manager.constraint import ( ComponentConstraints, - PackageConstraint, - VersionConstraint + PackageConstraint ) from sonic_package_manager.errors import ManifestError -from sonic_package_manager.version import Version, VersionRange +from sonic_package_manager.version import Version class ManifestSchema: From f5ba2462be37cfcd8b98af2dba910d9af75adec3 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 9 Apr 2021 13:40:00 +0300 Subject: [PATCH 27/60] fix review comments Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 2 +- sonic-utilities-data/bash_completion.d/spm | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) mode change 100644 => 120000 sonic-utilities-data/bash_completion.d/spm diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 23095a3fc5..a05605c60c 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7963,7 +7963,7 @@ Go Back To [Beginning of the document](#) or [Beginning of this section](#waterm ## Software Installation and Management SONiC image can be installed in one of two methods: -1. From within a running SONiC iamge using the `sonic-installer` utility +1. From within a running SONiC image using the `sonic-installer` utility 2. From the vendor's bootloader (E.g., ONIE, Aboot, etc.) SONiC feature Docker images (aka "SONiC packages") available to be installed with *sonic-package-manager* utility. diff --git a/sonic-utilities-data/bash_completion.d/spm b/sonic-utilities-data/bash_completion.d/spm deleted file mode 100644 index 8931dc389c..0000000000 --- a/sonic-utilities-data/bash_completion.d/spm +++ /dev/null @@ -1,8 +0,0 @@ -_sonic_package_manager_completion() { - COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ - COMP_CWORD=$COMP_CWORD \ - _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) ) - return 0 -} - -complete -F _sonic_package_manager_completion -o default spm; diff --git a/sonic-utilities-data/bash_completion.d/spm b/sonic-utilities-data/bash_completion.d/spm new file mode 120000 index 0000000000..3fff069223 --- /dev/null +++ b/sonic-utilities-data/bash_completion.d/spm @@ -0,0 +1 @@ +sonic-package-manager \ No newline at end of file From 2dee5c686b2a8aeae701276e640a33cee99568e4 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 14 Apr 2021 17:27:08 +0300 Subject: [PATCH 28/60] fix review comments Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 24 +++++++++++---------- sonic_package_manager/manager.py | 16 +++++++------- tests/sonic_package_manager/test_manager.py | 4 ++-- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 713113da61..025d9d9ff3 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -109,11 +109,11 @@ def handle_parse_result(self, ctx, opts, args): PACKAGE_COMMON_INSTALL_OPTIONS = [ - click.option('--skip-cli-plugin-installation', + click.option('--skip-host-plugins', is_flag=True, - help='Do not install CLI plugins provided by the package ' - 'on the host OS. NOTE: In case when package /cli/mandatory ' - 'field is set to True this option will fail the installation.'), + help='Do not install host OS plugins provided by the package (CLI, etc). ' + 'NOTE: In case when package host OS plugins are set as mandatory in ' + 'package manifest this option will fail the installation.'), ] @@ -286,7 +286,9 @@ def changelog(ctx, @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): - """ Add a new repository to database. """ + """ Add a new repository to database. + Repository in Docker Registry V2. + """ manager: PackageManager = ctx.obj @@ -338,8 +340,8 @@ def install(ctx, yes, enable, default_owner, - skip_cli_plugin_installation): - """ Install package """ + skip_host_plugins): + """ Install package using [PACKAGE_EXPR] in format "==" """ manager: PackageManager = ctx.obj @@ -353,7 +355,7 @@ def install(ctx, 'force': force, 'enable': enable, 'default_owner': default_owner, - 'skip_cli_plugin_installation': skip_cli_plugin_installation, + 'skip_host_plugins': skip_host_plugins, } try: @@ -379,8 +381,8 @@ def upgrade(ctx, from_tarball, force, yes, - skip_cli_plugin_installation): - """ Upgrade package """ + skip_host_plugins): + """ Upgrade package using [PACKAGE_EXPR] in format "==" """ manager: PackageManager = ctx.obj @@ -392,7 +394,7 @@ def upgrade(ctx, upgrade_opts = { 'force': force, - 'skip_cli_plugin_installation': skip_cli_plugin_installation, + 'skip_host_plugins': skip_host_plugins, } try: diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index f1643a1846..f0ff63add7 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -302,7 +302,7 @@ def install_from_source(self, force=False, enable=False, default_owner='local', - skip_cli_plugin_installation=False): + skip_host_plugins=False): """ Install SONiC Package from source represented by PackageSource. This method contains the logic of package installation. @@ -311,7 +311,7 @@ def install_from_source(self, force: Force the installation. enable: If True the installed feature package will be enabled. default_owner: Owner of the installed package. - skip_cli_plugin_installation: Skip CLI plugin installation. + skip_host_plugins: Skip CLI plugin installation. Raises: PackageManagerError """ @@ -330,7 +330,7 @@ def install_from_source(self, with failure_ignore(force): validate_package_base_os_constraints(package, self.version_info) validate_package_tree(installed_packages) - validate_package_cli_can_be_skipped(package, skip_cli_plugin_installation) + validate_package_cli_can_be_skipped(package, skip_host_plugins) # After all checks are passed we proceed to actual installation @@ -347,7 +347,7 @@ def install_from_source(self, self.service_creator.create(package, state=feature_state, owner=default_owner) exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) - if not skip_cli_plugin_installation: + if not skip_host_plugins: self._install_cli_plugins(package) exit_stack.callback(rollback_wrapper(self._uninstall_cli_plugins, package)) @@ -442,7 +442,7 @@ def upgrade(self, def upgrade_from_source(self, source: PackageSource, force=False, - skip_cli_plugin_installation=False): + skip_host_plugins=False): """ Upgrade SONiC Package to a version the package reference expression specifies. Can force the upgrade if force parameter is True. Force can allow a package downgrade. @@ -450,7 +450,7 @@ def upgrade_from_source(self, Args: source: SONiC Package source force: Force the upgrade. - skip_cli_plugin_installation: Skip CLI plugin installation. + skip_host_plugins: Skip host OS plugins installation. Raises: PackageManagerError """ @@ -492,7 +492,7 @@ def upgrade_from_source(self, with failure_ignore(force): validate_package_base_os_constraints(new_package, self.version_info) validate_package_tree(installed_packages) - validate_package_cli_can_be_skipped(new_package, skip_cli_plugin_installation) + validate_package_cli_can_be_skipped(new_package, skip_host_plugins) # After all checks are passed we proceed to actual upgrade @@ -528,7 +528,7 @@ def upgrade_from_source(self, if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') - if not skip_cli_plugin_installation: + if not skip_host_plugins: self._install_cli_plugins(new_package) exit_stack.pop_all() diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index 256832118e..901f3a89fe 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -164,7 +164,7 @@ def test_installation_cli_plugin_skipped(package_manager, fake_metadata_resolver manifest = fake_metadata_resolver.metadata_store['Azure/docker-test']['1.6.0']['manifest'] manifest['cli']= {'show': '/cli/plugin.py'} package_manager._install_cli_plugins = Mock() - package_manager.install('test-package', skip_cli_plugin_installation=True) + package_manager.install('test-package', skip_host_plugins=True) package_manager._install_cli_plugins.assert_not_called() @@ -174,7 +174,7 @@ def test_installation_cli_plugin_is_mandatory_but_skipped(package_manager, fake_ with pytest.raises(PackageManagerError, match='CLI is mandatory for package test-package but ' 'it was requested to be not installed'): - package_manager.install('test-package', skip_cli_plugin_installation=True) + package_manager.install('test-package', skip_host_plugins=True) def test_installation(package_manager, mock_docker_api, anything): From 08c9c6d4c2cc5554bbd583d9128c607b760653ec Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 15 Apr 2021 12:25:28 +0300 Subject: [PATCH 29/60] [sonic_package_manager] add reset command Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 137 +++++++++++--------- sonic_package_manager/main.py | 27 +++- sonic_package_manager/manager.py | 39 +++++- sonic_package_manager/source.py | 16 +-- tests/sonic_package_manager/conftest.py | 2 +- tests/sonic_package_manager/test_manager.py | 22 ++-- 6 files changed, 150 insertions(+), 93 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index a05605c60c..7c7c58e8a5 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8010,14 +8010,14 @@ This command will add a new entry in the package database. The package has to be - Usage: ``` - Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY +Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY - Add a new repository to database. + Add a new repository to database. Repository in Docker Registry V2. - Options: - --default-reference TEXT Default installation reference. - --description TEXT Optional package entry description. - --help Show this message and exit. +Options: + --default-reference TEXT Default installation reference + --description TEXT Optional package entry description + --help Show this message and exit. ``` - Example: ``` @@ -8049,41 +8049,34 @@ This command pulls and installs package on SONiC host. *NOTE*: this command requ - Usage: ``` - Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - - Install package - - Options: - --enable Set the default state of the feature to - enabled and enable feature right after - installation. NOTE: user needs to execute - "config save -y" to make this setting - persistent - - --default-owner [local|kube] Default owner configuration setting for a - feature - - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually - exclusive with arguments: [from_tarball, - package_expr]. +Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - --from-tarball FILE Fetch package from saved image tarball - NOTE: This argument is mutually exclusive - with arguments: [from_repository, - package_expr]. + Install package using [PACKAGE_EXPR] in format "==" - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or - DEBUG - - --skip-cli-plugin-installation Do not install CLI plugins provided by the - package on the host OS. NOTE: In case when - package /cli/mandatory field is set to True - this option will fail the installation. - - --help Show this message and exit. +Options: + --enable Set the default state of the feature to + enabled and enable feature right after + installation. NOTE: user needs to execute + "config save -y" to make this setting + persistent + --default-owner [local|kube] Default owner configuration setting for a + feature + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually + exclusive with arguments: [package_expr, + from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: + This argument is mutually exclusive with + arguments: [package_expr, from_repository]. + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package + host OS plugins are set as mandatory in + package manifest this option will fail the + installation. + --help Show this message and exit. ``` - Example: ``` @@ -8124,32 +8117,25 @@ This command upgrades package on SONiC host to a newer version. The procedure of - Usage: ``` - Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] - - Upgrade package - - Options: - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually - exclusive with arguments: [package_expr, - from_tarball]. - - --from-tarball FILE Fetch package from saved image tarball - NOTE: This argument is mutually exclusive - with arguments: [package_expr, - from_repository]. +Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or - DEBUG + Upgrade package using [PACKAGE_EXPR] in format "==" - --skip-cli-plugin-installation Do not install CLI plugins provided by the - package on the host OS. NOTE: In case when - package /cli/mandatory field is set to True - this option will fail the installation. - - --help Show this message and exit. +Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually exclusive + with arguments: [package_expr, from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: This + argument is mutually exclusive with arguments: + [from_repository, package_expr]. + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package host + OS plugins are set as mandatory in package manifest + this option will fail the installation. + --help Show this message and exit. ``` - Example: ``` @@ -8162,6 +8148,31 @@ This command upgrades package on SONiC host to a newer version. The procedure of admin@sonic:~$ sudo sonic-package-manager upgrade --from-tarball sonic-docker-image.gz ``` +**sonic-package-manager reset** + +This comamnd resets the package by reinstalling it to its default version. + +- Usage: + ``` +Usage: sonic-package-manager reset [OPTIONS] NAME + + Reset package to the default version + +Options: + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the package + (CLI, etc). NOTE: In case when package host OS plugins + are set as mandatory in package manifest this option + will fail the installation. + --help Show this message and exit. + ``` +- Example: + ``` + admin@sonic:~$ sudo sonic-package-manager reset dhcp-relay + ``` + **sonic-package-manager show package versions** This command will access repository for corresponding package and retrieve a list of available versions. diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 025d9d9ff3..13a59e8edb 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -281,8 +281,8 @@ def changelog(ctx, @repository.command() @click.argument('name', type=str) @click.argument('repository', type=str) -@click.option('--default-reference', type=str, help='Default installation reference.') -@click.option('--description', type=str, help='Default installation reference.') +@click.option('--default-reference', type=str, help='Default installation reference') +@click.option('--description', type=str, help='Optional package entry description') @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): @@ -408,6 +408,29 @@ def upgrade(ctx, exit_cli(f'Operation canceled by user', fg='red') +@cli.command() +@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) +@add_options(PACKAGE_COMMON_INSTALL_OPTIONS) +@click.argument('name') +@click.pass_context +@root_privileges_required +def reset(ctx, name, force, yes, skip_host_plugins): + """ Reset package to the default version """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm(f'Package {name} is going to be reset to default version, ' + f'continue?', abort=True, show_default=True) + + try: + manager.reset(name, force, skip_host_plugins) + except Exception as err: + exit_cli(f'Failed to reset package {name}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + @cli.command() @add_options(PACKAGE_COMMON_OPERATION_OPTIONS) @click.argument('name') diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index f0ff63add7..4077f875eb 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -442,7 +442,8 @@ def upgrade(self, def upgrade_from_source(self, source: PackageSource, force=False, - skip_host_plugins=False): + skip_host_plugins=False, + allow_downgrade=False): """ Upgrade SONiC Package to a version the package reference expression specifies. Can force the upgrade if force parameter is True. Force can allow a package downgrade. @@ -451,6 +452,7 @@ def upgrade_from_source(self, source: SONiC Package source force: Force the upgrade. skip_host_plugins: Skip host OS plugins installation. + allow_downgrade: Flag to allow package downgrade. Raises: PackageManagerError """ @@ -478,11 +480,9 @@ def upgrade_from_source(self, # TODO: Not all packages might support downgrade. # We put a check here but we understand that for some packages - # the downgrade might be safe to do. In that case we might want to - # add another argument to this function: allow_downgrade: bool = False. - # Another way to do that might be a variable in manifest describing package - # downgrade ability or downgrade-able versions. - if new_version < old_version: + # the downgrade might be safe to do. There can be a variable in manifest + # describing package downgrade ability or downgrade-able versions. + if new_version < old_version and not allow_downgrade: raise PackageUpgradeError(f'Request to downgrade from {old_version} to {new_version}. ' f'Downgrade might be not supported by the package') @@ -543,6 +543,33 @@ def upgrade_from_source(self, self.database.update_package(new_package_entry) self.database.commit() + @under_lock + def reset(self, name: str, force: bool = False, skip_host_plugins: bool = False): + """ Reset package to defaults version + + Args: + name: SONiC Package name. + force: Force the installation. + skip_host_plugins: Skip host plugins installation. + Raises: + PackageManagerError + """ + + with failure_ignore(force): + if not self.is_installed(name): + raise PackageManagerError(f'{name} is not installed') + + package = self.get_installed_package(name) + default_reference = package.entry.default_reference + if default_reference is None: + raise PackageManagerError(f'package {name} has no default reference') + + package_ref = PackageReference(name, default_reference) + source = self.get_package_source(package_ref=package_ref) + self.upgrade_from_source(source, force=force, + allow_downgrade=True, + skip_host_plugins=skip_host_plugins) + @under_lock def migrate_packages(self, old_package_database: PackageDatabase, diff --git a/sonic_package_manager/source.py b/sonic_package_manager/source.py index 4720b5be9e..40d2408ba7 100644 --- a/sonic_package_manager/source.py +++ b/sonic_package_manager/source.py @@ -76,20 +76,20 @@ def get_package(self) -> Package: name = manifest['package']['name'] description = manifest['package']['description'] + # Will be resolved in install() method. + # When installing from tarball we don't know yet + # the repository for this package. repository = None if self.database.has_package(name): # inherit package database info - package = self.database.get_package(name) - repository = package.repository - description = description or package.description + package_entry = self.database.get_package(name) + else: + package_entry = PackageEntry(name, repository, + description=description) return Package( - PackageEntry( - name, - repository, - description, - ), + package_entry, metadata ) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index d29f449684..cee997596c 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -29,7 +29,7 @@ def attrs(self): return {'RepoTags': [self.id]} def pull(repo, ref): - return Image(f'{repo}:latest') + return Image(f'{repo}:{ref}') def load(filename): return Image(filename) diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index 901f3a89fe..08b957475a 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -269,19 +269,6 @@ def test_installation_fault(package_manager, mock_docker_api, mock_service_creat mock_docker_api.rmi.assert_called_once() -def test_installation_package_with_description(package_manager, fake_metadata_resolver): - package_entry = package_manager.database.get_package('test-package') - description = package_entry.description - references = fake_metadata_resolver.metadata_store[package_entry.repository] - manifest = references[package_entry.default_reference]['manifest'] - new_description = description + ' changed description ' - manifest['package']['description'] = new_description - package_manager.install('test-package') - package_entry = package_manager.database.get_package('test-package') - description = package_entry.description - assert description == new_description - - def test_manager_installation_version_range(package_manager): with pytest.raises(PackageManagerError, match='Can only install specific version. ' @@ -298,6 +285,15 @@ def test_manager_upgrade(package_manager, sonic_fs): assert upgraded_package.entry.version == Version(2, 0, 0) +def test_manager_package_reset(package_manager, sonic_fs): + package_manager.install('test-package-6==1.5.0') + package_manager.upgrade('test-package-6==2.0.0') + + package_manager.reset('test-package-6') + upgraded_package = package_manager.get_installed_package('test-package-6') + assert upgraded_package.entry.version == Version(1, 5, 0) + + def test_manager_migration(package_manager, fake_db_for_migration): package_manager.install = Mock() package_manager.upgrade = Mock() From 413b44f092d8c99bd797fcf27a72d4adfde7ef81 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 16 Apr 2021 17:09:06 +0300 Subject: [PATCH 30/60] [doc] fix formatting Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 126 +++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 7c7c58e8a5..18c9e7d252 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7988,7 +7988,7 @@ SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-I Name Repository Description Version Status -------------- --------------------------- ---------------------------- --------- --------- database docker-database SONiC database package 1.0.0 Built-In - dhcp-relay docker-dhcp-relay SONiC dhcp-relay package 1.0.0 Installed + dhcp-relay azure/docker-dhcp-relay SONiC dhcp-relay package 1.0.0 Installed fpm-frr docker-fpm-frr SONiC fpm-frr package 1.0.0 Built-In lldp docker-lldp SONiC lldp package 1.0.0 Built-In macsec docker-macsec SONiC macsec package 1.0.0 Built-In @@ -8006,18 +8006,18 @@ SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-I **sonic-package-manager repository add** -This command will add a new entry in the package database. The package has to be *Not Installed* in order to be removed from package database. *NOTE*: this command requires elevated (root) privileges to run. +This command will add a new entry in the package database. *NOTE*: this command requires elevated (root) privileges to run. - Usage: ``` -Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY + Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY - Add a new repository to database. Repository in Docker Registry V2. + Add a new repository to database. Repository in Docker Registry V2. -Options: - --default-reference TEXT Default installation reference - --description TEXT Optional package entry description - --help Show this message and exit. + Options: + --default-reference TEXT Default installation reference + --description TEXT Optional package entry description + --help Show this message and exit. ``` - Example: ``` @@ -8027,7 +8027,7 @@ Options: **sonic-package-manager repository remove** -This command will remove an entry from the package database. *NOTE*: this command requires elevated (root) privileges to run. +This command will remove an entry from the package database. The package has to be *Not Installed* in order to be removed from package database. *NOTE*: this command requires elevated (root) privileges to run. - Usage: ``` @@ -8049,34 +8049,34 @@ This command pulls and installs package on SONiC host. *NOTE*: this command requ - Usage: ``` -Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] + Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - Install package using [PACKAGE_EXPR] in format "==" + Install package using [PACKAGE_EXPR] in format "==" -Options: - --enable Set the default state of the feature to - enabled and enable feature right after - installation. NOTE: user needs to execute - "config save -y" to make this setting - persistent - --default-owner [local|kube] Default owner configuration setting for a - feature - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually - exclusive with arguments: [package_expr, - from_tarball]. - --from-tarball FILE Fetch package from saved image tarball NOTE: - This argument is mutually exclusive with - arguments: [package_expr, from_repository]. - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the - package (CLI, etc). NOTE: In case when package - host OS plugins are set as mandatory in - package manifest this option will fail the - installation. - --help Show this message and exit. + Options: + --enable Set the default state of the feature to + enabled and enable feature right after + installation. NOTE: user needs to execute + "config save -y" to make this setting + persistent + --default-owner [local|kube] Default owner configuration setting for a + feature + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually + exclusive with arguments: [package_expr, + from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: + This argument is mutually exclusive with + arguments: [package_expr, from_repository]. + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package + host OS plugins are set as mandatory in + package manifest this option will fail the + installation. + --help Show this message and exit. ``` - Example: ``` @@ -8117,25 +8117,25 @@ This command upgrades package on SONiC host to a newer version. The procedure of - Usage: ``` -Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] + Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] - Upgrade package using [PACKAGE_EXPR] in format "==" + Upgrade package using [PACKAGE_EXPR] in format "==" -Options: - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually exclusive - with arguments: [package_expr, from_tarball]. - --from-tarball FILE Fetch package from saved image tarball NOTE: This - argument is mutually exclusive with arguments: - [from_repository, package_expr]. - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the - package (CLI, etc). NOTE: In case when package host - OS plugins are set as mandatory in package manifest - this option will fail the installation. - --help Show this message and exit. + Options: + --from-repository TEXT Fetch package directly from image registry + repository NOTE: This argument is mutually exclusive + with arguments: [package_expr, from_tarball]. + --from-tarball FILE Fetch package from saved image tarball NOTE: This + argument is mutually exclusive with arguments: + [from_repository, package_expr]. + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package host + OS plugins are set as mandatory in package manifest + this option will fail the installation. + --help Show this message and exit. ``` - Example: ``` @@ -8154,19 +8154,19 @@ This comamnd resets the package by reinstalling it to its default version. - Usage: ``` -Usage: sonic-package-manager reset [OPTIONS] NAME + Usage: sonic-package-manager reset [OPTIONS] NAME - Reset package to the default version + Reset package to the default version -Options: - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the package - (CLI, etc). NOTE: In case when package host OS plugins - are set as mandatory in package manifest this option - will fail the installation. - --help Show this message and exit. + Options: + -f, --force Force operation by ignoring failures + -y, --yes Automatically answer yes on prompts + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + --skip-host-plugins Do not install host OS plugins provided by the package + (CLI, etc). NOTE: In case when package host OS plugins + are set as mandatory in package manifest this option + will fail the installation. + --help Show this message and exit. ``` - Example: ``` From f101f1f97c49d6ee4456a4ccbae950b9de85476f Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 16 Apr 2021 17:18:40 +0300 Subject: [PATCH 31/60] install takes a digest or tag as well Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 18c9e7d252..b85a368132 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8051,7 +8051,7 @@ This command pulls and installs package on SONiC host. *NOTE*: this command requ ``` Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - Install package using [PACKAGE_EXPR] in format "==" + Install package using [PACKAGE_EXPR] in format "==" or "@" Options: --enable Set the default state of the feature to From 4d03aa89f66618b8b5096e14c045b0c213a3c016 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 20 Apr 2021 17:09:24 +0300 Subject: [PATCH 32/60] resolve review comments Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 71 +++----- sonic_package_manager/main.py | 46 +---- sonic_package_manager/manager.py | 183 ++++++++++---------- sonic_package_manager/source.py | 15 +- tests/sonic_package_manager/test_manager.py | 42 +++-- 5 files changed, 161 insertions(+), 196 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index b85a368132..01ab77acce 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7986,7 +7986,8 @@ SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-I ``` admin@sonic:~$ sonic-package-manager list Name Repository Description Version Status - -------------- --------------------------- ---------------------------- --------- --------- + -------------- --------------------------- ---------------------------- --------- -------------- + cpu-report azure/cpu-report CPU report package N/A Not Installed database docker-database SONiC database package 1.0.0 Built-In dhcp-relay azure/docker-dhcp-relay SONiC dhcp-relay package 1.0.0 Installed fpm-frr docker-fpm-frr SONiC fpm-frr package 1.0.0 Built-In @@ -8045,13 +8046,15 @@ This command will remove an entry from the package database. The package has to **sonic-package-manager install** -This command pulls and installs package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run. +This command pulls and installs or upgrades (if already installed) package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run. +The procedure of upgrading a package will restart the corresponding service. - Usage: ``` Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] - Install package using [PACKAGE_EXPR] in format "==" or "@" + Install/Upgrade package using [PACKAGE_EXPR] in format + "[=|@]" Options: --enable Set the default state of the feature to @@ -8080,7 +8083,13 @@ This command pulls and installs package on SONiC host. *NOTE*: this command requ ``` - Example: ``` - admin@sonic:~$ sudo sonic-package-manager install dhcp-relay==1.0.2 + admin@sonic:~$ sudo sonic-package-manager install dhcp-relay=1.0.2 + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager install dhcp-relay@latest + ``` + ``` + admin@sonic:~$ sudo sonic-package-manager install dhcp-relay@sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd ``` ``` admin@sonic:~$ sudo sonic-package-manager install --from-repository azure/sonic-cpu-report:latest @@ -8111,46 +8120,9 @@ This command uninstalls package from SONiC host. *NOTE*: this command requires e admin@sonic:~$ sudo sonic-package-manager uninstall dhcp-relay ``` -**sonic-package-manager upgrade** - -This command upgrades package on SONiC host to a newer version. The procedure of upgrading a package will restart the corresponding service. *NOTE*: this command requires elevated (root) privileges to run. - -- Usage: - ``` - Usage: sonic-package-manager upgrade [OPTIONS] [PACKAGE_EXPR] - - Upgrade package using [PACKAGE_EXPR] in format "==" - - Options: - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually exclusive - with arguments: [package_expr, from_tarball]. - --from-tarball FILE Fetch package from saved image tarball NOTE: This - argument is mutually exclusive with arguments: - [from_repository, package_expr]. - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the - package (CLI, etc). NOTE: In case when package host - OS plugins are set as mandatory in package manifest - this option will fail the installation. - --help Show this message and exit. - ``` -- Example: - ``` - admin@sonic:~$ sudo sonic-package-manager upgrade dhcp-relay==2.0.0 - ``` - ``` - admin@sonic:~$ sudo sonic-package-manager upgrade --from-repository azure/sonic-cpu-report:latest - ``` - ``` - admin@sonic:~$ sudo sonic-package-manager upgrade --from-tarball sonic-docker-image.gz - ``` - **sonic-package-manager reset** -This comamnd resets the package by reinstalling it to its default version. +This comamnd resets the package by reinstalling it to its default version. *NOTE*: this command requires elevated (root) privileges to run. - Usage: ``` @@ -8195,6 +8167,19 @@ This command will access repository for corresponding package and retrieve a lis • 1.0.2 • 2.0.0 ``` + ``` + admin@sonic:~$ sonic-package-manager show package versions dhcp-relay --plain + 1.0.0 + 1.0.2 + 2.0.0 + ``` + ``` + admin@sonic:~$ sonic-package-manager show package versions dhcp-relay --all + • 1.0.0 + • 1.0.2 + • 2.0.0 + • latest + ``` **sonic-package-manager show package changelog** @@ -8247,7 +8232,7 @@ This command fetches the package manifest and displays it. *NOTE*: package manif ``` - Example: ``` - admin@sonic:~$ sonic-package-manager show package manifest dhcp-relay==2.0.0 + admin@sonic:~$ sonic-package-manager show package manifest dhcp-relay=2.0.0 { "version": "1.0.0", "package": { diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 13a59e8edb..1f2d5011d4 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -174,7 +174,7 @@ def package(ctx): @cli.command() @click.pass_context def list(ctx): - """ List available repositories """ + """ List available packages """ table_header = ['Name', 'Repository', 'Description', 'Version', 'Status'] table_body = [] @@ -341,11 +341,14 @@ def install(ctx, enable, default_owner, skip_host_plugins): - """ Install package using [PACKAGE_EXPR] in format "==" """ + """ Install/Upgrade package using [PACKAGE_EXPR] in format + "[=|@]" """ manager: PackageManager = ctx.obj package_source = package_expr or from_repository or from_tarball + if not package_source: + exit_cli(f'Package source is not specified', fg='red') if not yes and not force: click.confirm(f'{package_source} is going to be installed, ' @@ -369,45 +372,6 @@ def install(ctx, exit_cli(f'Operation canceled by user', fg='red') -@cli.command() -@add_options(PACKAGE_SOURCE_OPTIONS) -@add_options(PACKAGE_COMMON_OPERATION_OPTIONS) -@add_options(PACKAGE_COMMON_INSTALL_OPTIONS) -@click.pass_context -@root_privileges_required -def upgrade(ctx, - package_expr, - from_repository, - from_tarball, - force, - yes, - skip_host_plugins): - """ Upgrade package using [PACKAGE_EXPR] in format "==" """ - - manager: PackageManager = ctx.obj - - package_source = package_expr or from_repository or from_tarball - - if not yes and not force: - click.confirm(f'Package is going to be upgraded with {package_source}, ' - f'continue?', abort=True, show_default=True) - - upgrade_opts = { - 'force': force, - 'skip_host_plugins': skip_host_plugins, - } - - try: - manager.upgrade(package_expr, - from_repository, - from_tarball, - **upgrade_opts) - except Exception as err: - exit_cli(f'Failed to upgrade {package_source}: {err}', fg='red') - except KeyboardInterrupt: - exit_cli(f'Operation canceled by user', fg='red') - - @cli.command() @add_options(PACKAGE_COMMON_OPERATION_OPTIONS) @add_options(PACKAGE_COMMON_INSTALL_OPTIONS) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 4077f875eb..281f890bcc 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -4,6 +4,7 @@ import os import pkgutil import tempfile +from inspect import signature from typing import Any, Iterable, Callable, Dict, Optional import docker @@ -37,7 +38,10 @@ from sonic_package_manager.progress import ProgressManager from sonic_package_manager.reference import PackageReference from sonic_package_manager.registry import RegistryResolver -from sonic_package_manager.service_creator.creator import ServiceCreator, run_command +from sonic_package_manager.service_creator.creator import ( + ServiceCreator, + run_command +) from sonic_package_manager.service_creator.feature import FeatureRegistry from sonic_package_manager.service_creator.sonic_db import SonicDB from sonic_package_manager.service_creator.utils import in_chroot @@ -51,7 +55,8 @@ from sonic_package_manager.version import ( Version, VersionRange, - version_to_tag, tag_to_version + version_to_tag, + tag_to_version ) @@ -80,7 +85,23 @@ def wrapped_function(*args, **kwargs): return wrapped_function -def rollback_wrapper(func, *args, **kwargs): +def opt_check(func: Callable) -> Callable: + """ Check kwargs for function. """ + + @functools.wraps(func) + def wrapped_function(*args, **kwargs): + sig = signature(func) + redundant_opts = [opt for opt in kwargs if opt not in sig.parameters] + if redundant_opts: + raise PackageManagerError( + f'Unsupported options: {",".join(redundant_opts)} for {func.__name__}' + ) + return func(*args, **kwargs) + + return wrapped_function + + +def rollback(func, *args, **kwargs): """ Used in rollback callbacks to ignore failure but proceed with rollback. Error will be printed but not fail the whole procedure of rollback. """ @@ -104,7 +125,7 @@ def package_constraint_to_reference(constraint: PackageConstraint) -> PackageRef return PackageReference(package_name, None) if not isinstance(version_constraint, Version): raise PackageManagerError(f'Can only install specific version. ' - f'Use only following expression "{package_name}==" ' + f'Use only following expression "{package_name}=" ' f'to install specific version') return PackageReference(package_name, version_to_tag(version_constraint)) @@ -280,23 +301,29 @@ def install(self, repotag: Optional[str] = None, tarball: Optional[str] = None, **kwargs): - """ Install SONiC Package from either an expression representing - the package and its version, repository and tag or digest in - same format as "docker pulL" accepts or an image tarball path. + """ Install/Upgrade SONiC Package from either an expression + representing the package and its version, repository and tag or + digest in same format as "docker pulL" accepts or an image tarball path. Args: expression: SONiC Package reference expression - repotag: Install from REPO[:TAG][@DIGEST] - tarball: Install from tarball, path to tarball file - kwargs: Install options for self.install_from_source + repotag: Install/Upgrade from REPO[:TAG][@DIGEST] + tarball: Install/Upgrade from tarball, path to tarball file + kwargs: Install/Upgrade options for self.install_from_source Raises: PackageManagerError """ source = self.get_package_source(expression, repotag, tarball) - self.install_from_source(source, **kwargs) + package = source.get_package() + + if self.is_installed(package.name): + self.upgrade_from_source(source, **kwargs) + else: + self.install_from_source(source, **kwargs) @under_lock + @opt_check def install_from_source(self, source: PackageSource, force=False, @@ -340,18 +367,18 @@ def install_from_source(self, self.database.add_package(package.name, package.repository) try: - with contextlib.ExitStack() as exit_stack: + with contextlib.ExitStack() as exits: source.install(package) - exit_stack.callback(rollback_wrapper(source.uninstall, package)) + exits.callback(rollback(source.uninstall, package)) self.service_creator.create(package, state=feature_state, owner=default_owner) - exit_stack.callback(rollback_wrapper(self.service_creator.remove, package)) + exits.callback(rollback(self.service_creator.remove, package)) if not skip_host_plugins: self._install_cli_plugins(package) - exit_stack.callback(rollback_wrapper(self._uninstall_cli_plugins, package)) + exits.callback(rollback(self._uninstall_cli_plugins, package)) - exit_stack.pop_all() + exits.pop_all() except Exception as err: raise PackageInstallationError(f'Failed to install {package.name}: {err}') except KeyboardInterrupt: @@ -363,6 +390,7 @@ def install_from_source(self, self.database.commit() @under_lock + @opt_check def uninstall(self, name: str, force=False): """ Uninstall SONiC Package referenced by name. The uninstallation can be forced if force argument is True. @@ -381,6 +409,8 @@ def uninstall(self, name: str, force=False): package = self.get_installed_package(name) service_name = package.manifest['service']['name'] + name = package.name + with failure_ignore(force): if self.feature_registry.is_feature_enabled(service_name): raise PackageUninstallationError( @@ -402,14 +432,17 @@ def uninstall(self, name: str, force=False): self.service_creator.remove(package) # Clean containers based on this image - containers = self.docker.ps(filters={'ancestor': package.image_id}, all=True) + containers = self.docker.ps(filters={'ancestor': package.image_id}, + all=True) for container in containers: self.docker.rm(container.id, force=True) self.docker.rmi(package.image_id, force=True) package.entry.image_id = None except Exception as err: - raise PackageUninstallationError(f'Failed to uninstall {package.name}: {err}') + raise PackageUninstallationError( + f'Failed to uninstall {package.name}: {err}' + ) package.entry.installed = False package.entry.version = None @@ -417,28 +450,7 @@ def uninstall(self, name: str, force=False): self.database.commit() @under_lock - def upgrade(self, - expression: Optional[str] = None, - repotag: Optional[str] = None, - tarball: Optional[str] = None, - **kwargs): - """ Upgrade SONiC Package from either an expression representing - the package and its version, repository and tag or digest in - same format as "docker pulL" accepts or an image tarball path. - - Args: - expression: SONiC Package reference expression - repotag: Upgrade from REPO[:TAG][@DIGEST] - tarball: Upgrade from tarball, path to tarball file - kwargs: Upgrade options for self.upgrade_from_source - Raises: - PackageManagerError - """ - - source = self.get_package_source(expression, repotag, tarball) - self.upgrade_from_source(source, **kwargs) - - @under_lock + @opt_check def upgrade_from_source(self, source: PackageSource, force=False, @@ -467,7 +479,9 @@ def upgrade_from_source(self, old_package = self.get_installed_package(name) if old_package.built_in: - raise PackageUpgradeError(f'Cannot upgrade built-in package {old_package.name}') + raise PackageUpgradeError( + f'Cannot upgrade built-in package {old_package.name}' + ) old_feature = old_package.manifest['service']['name'] new_feature = new_package.manifest['service']['name'] @@ -483,8 +497,10 @@ def upgrade_from_source(self, # the downgrade might be safe to do. There can be a variable in manifest # describing package downgrade ability or downgrade-able versions. if new_version < old_version and not allow_downgrade: - raise PackageUpgradeError(f'Request to downgrade from {old_version} to {new_version}. ' - f'Downgrade might be not supported by the package') + raise PackageUpgradeError( + f'Request to downgrade from {old_version} to {new_version}. ' + f'Downgrade might be not supported by the package' + ) # remove currently installed package from the list installed_packages = self._get_installed_packages_and(new_package) @@ -497,41 +513,42 @@ def upgrade_from_source(self, # After all checks are passed we proceed to actual upgrade try: - with contextlib.ExitStack() as exit_stack: + with contextlib.ExitStack() as exits: self._uninstall_cli_plugins(old_package) - exit_stack.callback(rollback_wrapper(self._install_cli_plugins, old_package)) + exits.callback(rollback(self._install_cli_plugins, old_package)) source.install(new_package) - exit_stack.callback(rollback_wrapper(source.uninstall, new_package)) + exits.callback(rollback(source.uninstall, new_package)) if self.feature_registry.is_feature_enabled(old_feature): self._systemctl_action(old_package, 'stop') - exit_stack.callback(rollback_wrapper(self._systemctl_action, + exits.callback(rollback(self._systemctl_action, old_package, 'start')) self.service_creator.remove(old_package, deregister_feature=False) - exit_stack.callback(rollback_wrapper(self.service_creator.create, - old_package, register_feature=False)) - - # This is no return point, after we start removing old Docker images - # there is no guaranty we can actually successfully roll-back. + exits.callback(rollback(self.service_creator.create, + old_package, register_feature=False)) # Clean containers based on the old image - containers = self.docker.ps(filters={'ancestor': old_package.image_id}, all=True) + containers = self.docker.ps(filters={'ancestor': old_package.image_id}, + all=True) for container in containers: self.docker.rm(container.id, force=True) - self.docker.rmi(old_package.image_id, force=True) - self.service_creator.create(new_package, register_feature=False) + exits.callback(rollback(self.service_creator.remove, new_package, + register_feature=False)) if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') if not skip_host_plugins: self._install_cli_plugins(new_package) + exits.callback(rollback(self._uninstall_cli_plugin, old_package)) - exit_stack.pop_all() + self.docker.rmi(old_package.image_id, force=True) + + exits.pop_all() except Exception as err: raise PackageUpgradeError(f'Failed to upgrade {new_package.name}: {err}') except KeyboardInterrupt: @@ -544,6 +561,7 @@ def upgrade_from_source(self, self.database.commit() @under_lock + @opt_check def reset(self, name: str, force: bool = False, skip_host_plugins: bool = False): """ Reset package to defaults version @@ -574,19 +592,18 @@ def reset(self, name: str, force: bool = False, skip_host_plugins: bool = False) def migrate_packages(self, old_package_database: PackageDatabase, dockerd_sock: Optional[str] = None): - """ Migrate packages from old database. This function can - do a comparison between current database and the database - passed in as argument. - If the package is missing in the current database it will be added. - If the package is installed in the passed database and in the current - it is not installed it will be installed with a passed database package version. - If the package is installed in the passed database and it is installed - in the current database but with older version the package will be upgraded to - the never version. - If the package is installed in the passed database and in the current - it is installed but with never version - no actions are taken. - If dockerd_sock parameter is passed, the migration process will use loaded - images from docker library of the currently installed image. + """ + Migrate packages from old database. This function can do a comparison between + current database and the database passed in as argument. If the package is + missing in the current database it will be added. If the package is installed + in the passed database and in the current it is not installed it will be + installed with a passed database package version. If the package is installed + in the passed database and it is installed in the current database but with + older version the package will be upgraded to the never version. If the package + is installed in the passed database and in the current it is installed but with + never version - no actions are taken. If dockerd_sock parameter is passed, the + migration process will use loaded images from docker library of the currently + installed image. Args: old_package_database: SONiC Package Database to migrate packages from. @@ -598,31 +615,21 @@ def migrate_packages(self, self._migrate_package_database(old_package_database) def migrate_package(old_package_entry, - new_package_entry, - migrate_operation=None): + new_package_entry): """ Migrate package routine Args: old_package_entry: Entry in old package database. new_package_entry: Entry in new package database. - migrate_operation: Operation to perform: install or upgrade. """ - try: - migrate_func = { - 'install': self.install, - 'upgrade': self.upgrade, - }[migrate_operation] - except KeyError: - raise ValueError(f'Invalid operation passed in {migrate_operation}') - name = new_package_entry.name version = new_package_entry.version if dockerd_sock: # dockerd_sock is defined, so use docked_sock to connect to # dockerd and fetch package image from it. - log.info(f'{migrate_operation} {name} from old docker library') + log.info(f'installing {name} from old docker library') docker_api = DockerApi(docker.DockerClient(base_url=f'unix://{dockerd_sock}')) image = docker_api.get_image(old_package_entry.image_id) @@ -631,11 +638,11 @@ def migrate_package(old_package_entry, for chunk in image.save(named=True): file.write(chunk) - migrate_func(tarball=file.name) + self.install(tarball=file.name) else: - log.info(f'{migrate_operation} {name} version {version}') + log.info(f'installing {name} version {version}') - migrate_func(f'{name}=={version}') + self.install(f'{name}={version}') # TODO: Topological sort packages by their dependencies first. for old_package in old_package_database: @@ -653,7 +660,7 @@ def migrate_package(old_package_entry, f'{old_package.version} > {new_package.version}') log.info(f'upgrading {new_package.name} to {old_package.version}') new_package.version = old_package.version - migrate_package(old_package, new_package, 'upgrade') + migrate_package(old_package, new_package) else: log.info(f'skipping {new_package.name} as installed version is newer') elif new_package.default_reference is not None: @@ -666,14 +673,14 @@ def migrate_package(old_package_entry, f'then the default in new image: ' f'{old_package.version} > {new_package_default_version}') new_package.version = old_package.version - migrate_package(old_package, new_package, 'install') + migrate_package(old_package, new_package) else: - self.install(f'{new_package.name}=={new_package_default_version}') + self.install(f'{new_package.name}={new_package_default_version}') else: # No default version and package is not installed. # Migrate old package same version. new_package.version = old_package.version - migrate_package(old_package, new_package, 'install') + migrate_package(old_package, new_package) self.database.commit() diff --git a/sonic_package_manager/source.py b/sonic_package_manager/source.py index 40d2408ba7..c179e0b3ee 100644 --- a/sonic_package_manager/source.py +++ b/sonic_package_manager/source.py @@ -28,10 +28,12 @@ def get_metadata(self) -> Metadata: """ raise NotImplementedError - def install_image(self): + def install_image(self, package: Package): """ Install image based on package source. Child class has to implement this method. + Args: + package: SONiC Package Returns: Docker Image object. """ @@ -46,7 +48,7 @@ def install(self, package: Package): package: SONiC Package """ - image = self.install_image() + image = self.install_image(package) package.entry.image_id = image.id # if no repository is defined for this package # get repository from image @@ -113,7 +115,7 @@ def get_metadata(self) -> Metadata: return self.metadata_resolver.from_tarball(self.tarball_path) - def install_image(self): + def install_image(self, package: Package): """ Installs image from local tarball source. """ return self.docker.load(self.tarball_path) @@ -141,10 +143,13 @@ def get_metadata(self) -> Metadata: return self.metadata_resolver.from_registry(self.repository, self.reference) - def install_image(self): + def install_image(self, package: Package): """ Installs image from registry. """ - return self.docker.pull(self.repository, self.reference) + image_id = self.docker.pull(self.repository, self.reference) + if not package.entry.default_reference: + package.entry.default_reference = self.reference + return image_id class LocalSource(PackageSource): diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index 08b957475a..dc79f7f483 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -9,12 +9,16 @@ def test_installation_not_installed(package_manager): package_manager.install('test-package') + package = package_manager.get_installed_package('test-package') + assert package.installed + assert package.entry.default_reference == '1.6.0' def test_installation_already_installed(package_manager): - with pytest.raises(PackageInstallationError, - match='swss is already installed'): - package_manager.install('swss') + package_manager.install('test-package') + with pytest.raises(PackageManagerError, + match='1.6.0 is already installed'): + package_manager.install('test-package') def test_installation_dependencies(package_manager, fake_metadata_resolver, mock_docker_api): @@ -197,7 +201,7 @@ def test_installation_using_reference(package_manager, def test_manager_installation_tag(package_manager, mock_docker_api, anything): - package_manager.install(f'test-package==1.6.0') + package_manager.install(f'test-package=1.6.0') mock_docker_api.pull.assert_called_once_with('Azure/docker-test', '1.6.0') @@ -240,10 +244,10 @@ def test_installation_from_file_unknown_package(package_manager, fake_db, sonic_ def test_upgrade_from_file_known_package(package_manager, fake_db, sonic_fs): repository = fake_db.get_package('test-package-6').repository # install older version from repository - package_manager.install('test-package-6==1.5.0') + package_manager.install('test-package-6=1.5.0') # upgrade from file sonic_fs.create_file('Azure/docker-test-6:2.0.0') - package_manager.upgrade(tarball='Azure/docker-test-6:2.0.0') + package_manager.install(tarball='Azure/docker-test-6:2.0.0') # locally installed package does not override already known package repository assert repository == fake_db.get_package('test-package-6').repository @@ -272,22 +276,24 @@ def test_installation_fault(package_manager, mock_docker_api, mock_service_creat def test_manager_installation_version_range(package_manager): with pytest.raises(PackageManagerError, match='Can only install specific version. ' - 'Use only following expression "test-package==" ' + 'Use only following expression "test-package=" ' 'to install specific version'): package_manager.install(f'test-package>=1.6.0') def test_manager_upgrade(package_manager, sonic_fs): - package_manager.install('test-package-6==1.5.0') - package_manager.upgrade('test-package-6==2.0.0') + package_manager.install('test-package-6=1.5.0') + package = package_manager.get_installed_package('test-package-6') + package_manager.install('test-package-6=2.0.0') upgraded_package = package_manager.get_installed_package('test-package-6') assert upgraded_package.entry.version == Version(2, 0, 0) + assert upgraded_package.entry.default_reference == package.entry.default_reference def test_manager_package_reset(package_manager, sonic_fs): - package_manager.install('test-package-6==1.5.0') - package_manager.upgrade('test-package-6==2.0.0') + package_manager.install('test-package-6=1.5.0') + package_manager.install('test-package-6=2.0.0') package_manager.reset('test-package-6') upgraded_package = package_manager.get_installed_package('test-package-6') @@ -296,22 +302,20 @@ def test_manager_package_reset(package_manager, sonic_fs): def test_manager_migration(package_manager, fake_db_for_migration): package_manager.install = Mock() - package_manager.upgrade = Mock() package_manager.migrate_packages(fake_db_for_migration) - # test-package-3 was installed but there is a newer version installed - # in fake_db_for_migration, asserting for upgrade - package_manager.upgrade.assert_has_calls([call('test-package-3==1.6.0')], any_order=True) - package_manager.install.assert_has_calls([ + # test-package-3 was installed but there is a newer version installed + # in fake_db_for_migration, asserting for upgrade + call('test-package-3=1.6.0'), # test-package-4 was not present in DB at all, but it is present and installed in # fake_db_for_migration, thus asserting that it is going to be installed. - call('test-package-4==1.5.0'), + call('test-package-4=1.5.0'), # test-package-5 1.5.0 was installed in fake_db_for_migration but the default # in current db is 1.9.0, assert that migration will install the newer version. - call('test-package-5==1.9.0'), + call('test-package-5=1.9.0'), # test-package-6 2.0.0 was installed in fake_db_for_migration but the default # in current db is 1.5.0, assert that migration will install the newer version. - call('test-package-6==2.0.0')], + call('test-package-6=2.0.0')], any_order=True ) From 180eae301434f004756d8c0772f32b70f18ee933 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 20 Apr 2021 19:26:19 +0300 Subject: [PATCH 33/60] remove config when uninstalling Signed-off-by: Stepan Blyschak --- .../service_creator/creator.py | 48 +++++++++++++------ .../test_service_creator.py | 4 +- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 53fa51075d..d86b70f979 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -146,6 +146,7 @@ def remove_file(path): if deregister_feature: self.feature_registry.deregister(package.manifest['service']['name']) + self.remove_config(package) def post_operation_hook(self): if not in_chroot(): @@ -287,31 +288,31 @@ def generate_dump_script(self, package): render_template(scrip_template, script_path, render_ctx, executable=True) log.info(f'generated {script_path}') - def set_initial_config(self, package): - init_cfg = package.manifest['package']['init-cfg'] + def get_tables(self, table_name): + tables = [] - def get_tables(table_name): - tables = [] + running_table = self.sonic_db.running_table(table_name) + if running_table is not None: + tables.append(running_table) - running_table = self.sonic_db.running_table(table_name) - if running_table is not None: - tables.append(running_table) + persistent_table = self.sonic_db.persistent_table(table_name) + if persistent_table is not None: + tables.append(persistent_table) - persistent_table = self.sonic_db.persistent_table(table_name) - if persistent_table is not None: - tables.append(persistent_table) + initial_table = self.sonic_db.initial_table(table_name) + if initial_table is not None: + tables.append(initial_table) - initial_table = self.sonic_db.initial_table(table_name) - if initial_table is not None: - tables.append(initial_table) + return tables - return tables + def set_initial_config(self, package): + init_cfg = package.manifest['package']['init-cfg'] for tablename, content in init_cfg.items(): if not isinstance(content, dict): continue - tables = get_tables(tablename) + tables = self.get_tables(tablename) for key in content: for table in tables: @@ -321,3 +322,20 @@ def get_tables(table_name): cfg.update(old_fvs) fvs = list(cfg.items()) table.set(key, fvs) + + def remove_config(self, package): + # Remove configuration based on init-cfg tables, so having + # init-cfg even with tables without keys might be a good idea. + # TODO: init-cfg should be validated with yang model + # TODO: remove config from tables known to yang model + init_cfg = package.manifest['package']['init-cfg'] + + for tablename, content in init_cfg.items(): + if not isinstance(content, dict): + continue + + tables = self.get_tables(tablename) + + for key in content: + for table in tables: + table._del(key) diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index f6ef317d8c..c540259b49 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -105,10 +105,12 @@ def test_service_creator_initial_config(sonic_fs, manifest, mock_feature_registr package = Package(entry, Metadata(manifest)) creator.create(package) - mock_table.set.assert_called_with('key_a', [('field_1', 'value_1'), ('field_2', 'original_value_2')]) + creator.remove(package) + mock_table._del.assert_called_with('key_a') + def test_feature_registration(mock_sonic_db, manifest): mock_feature_table = Mock() From 3b984891640073be70f2bab6c0aa38010b7f4803 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Tue, 20 Apr 2021 19:26:28 +0300 Subject: [PATCH 34/60] fix lgtm error Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 281f890bcc..2a6ea96b9c 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -409,8 +409,6 @@ def uninstall(self, name: str, force=False): package = self.get_installed_package(name) service_name = package.manifest['service']['name'] - name = package.name - with failure_ignore(force): if self.feature_registry.is_feature_enabled(service_name): raise PackageUninstallationError( From 3280b5b32d0195dda06d0fcf309b43dbe37c0ec9 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:55:58 +0300 Subject: [PATCH 35/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 01ab77acce..7caa0d891b 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7962,7 +7962,7 @@ Go Back To [Beginning of the document](#) or [Beginning of this section](#waterm ## Software Installation and Management -SONiC image can be installed in one of two methods: +SONiC images can be installed in one of two methods: 1. From within a running SONiC image using the `sonic-installer` utility 2. From the vendor's bootloader (E.g., ONIE, Aboot, etc.) From 21ae66be5553ff2ff02caa7c0c734b44936e7592 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:56:13 +0300 Subject: [PATCH 36/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 7caa0d891b..750c492f0a 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7966,7 +7966,7 @@ SONiC images can be installed in one of two methods: 1. From within a running SONiC image using the `sonic-installer` utility 2. From the vendor's bootloader (E.g., ONIE, Aboot, etc.) -SONiC feature Docker images (aka "SONiC packages") available to be installed with *sonic-package-manager* utility. +SONiC packages are available as prebuilt Docker images and meant to be installed with the *sonic-package-manager* utility. ### SONiC Package Manager From 55489765d959787e3d04383e91f3d28a8446051d Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:56:35 +0300 Subject: [PATCH 37/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 750c492f0a..466ae36dfc 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -7970,7 +7970,7 @@ SONiC packages are available as prebuilt Docker images and meant to be installed ### SONiC Package Manager -This is a command line tool that provides functionality to manage SONiC Packages on SONiC device. +The *sonic-package-manager* is a command line tool to manage (e.g. install, upgrade or uninstall) SONiC Packages. **sonic-package-manager list** From 5a2675dd9c84f8e6a0d75ca8d85dd7d2eb6259f9 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:57:07 +0300 Subject: [PATCH 38/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 466ae36dfc..c42f2c5bec 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8007,7 +8007,7 @@ SONiC package status can be *Installed*, *Not installed* or *Built-In*. "Built-I **sonic-package-manager repository add** -This command will add a new entry in the package database. *NOTE*: this command requires elevated (root) privileges to run. +This command will add a new repository as source for SONiC packages to the database. *NOTE*: requires elevated (root) privileges to run - Usage: ``` From 68f90f5b2c25562cd17e570905a7eec8838e3da0 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:57:27 +0300 Subject: [PATCH 39/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index c42f2c5bec..634853b8ce 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8013,7 +8013,7 @@ This command will add a new repository as source for SONiC packages to the datab ``` Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY - Add a new repository to database. Repository in Docker Registry V2. + Add new repository to database Options: --default-reference TEXT Default installation reference From e7f1c598e09418dd0f15b6b13a9906a10d473c53 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:57:50 +0300 Subject: [PATCH 40/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 634853b8ce..b47b908771 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8028,7 +8028,7 @@ This command will add a new repository as source for SONiC packages to the datab **sonic-package-manager repository remove** -This command will remove an entry from the package database. The package has to be *Not Installed* in order to be removed from package database. *NOTE*: this command requires elevated (root) privileges to run. +This command will remove a repository as source for SONiC packages from the database . The package has to be *Not Installed* in order to be removed from package database. *NOTE*: requires elevated (root) privileges to run - Usage: ``` From ae5bad9acb1b0384bf30c7c86c33dff7dff94856 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:59:17 +0300 Subject: [PATCH 41/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index b47b908771..c837cc3a54 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8034,7 +8034,7 @@ This command will remove a repository as source for SONiC packages from the data ``` Usage: sonic-package-manager repository remove [OPTIONS] NAME - Remove package from database. + Remove repository from database Options: --help Show this message and exit. From 909079a4ed2fc571335c6095c48e9b4bb0215765 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 03:59:44 +0300 Subject: [PATCH 42/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index c837cc3a54..1a3a6bc714 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8218,7 +8218,7 @@ This command fetches the package manifest and displays it. *NOTE*: package manif ``` Usage: sonic-package-manager show package manifest [OPTIONS] [PACKAGE_EXPR] - Print package manifest content + Show package manifest Options: --from-repository TEXT Fetch package directly from image registry From a394b27df3713cd84e4353f2972a527edf0bad79 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:00:12 +0300 Subject: [PATCH 43/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 1a3a6bc714..5fd79c7431 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8189,7 +8189,7 @@ This command fetches the changelog from package manifest and displays it. *NOTE* ``` Usage: sonic-package-manager show package changelog [OPTIONS] [PACKAGE_EXPR] - Print package changelog + Show package changelog Options: --from-repository TEXT Fetch package directly from image registry From 9653c616a6c75314497eb93c88374222bb75a78a Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:00:44 +0300 Subject: [PATCH 44/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 5fd79c7431..7ee47fe34f 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8316,7 +8316,7 @@ This command is used to install a new image on the alternate image partition. T Done ``` -SONiC image installation will install SONiC packages that are installed in currently running SONiC image. In order to perform clean SONiC installation use *--skip-package-migration* option when installing SONiC image: +Installing a new image using the sonic-installer will keep using the packages installed on the currently running SONiC image and automatically migrate those. In order to perform clean SONiC installation use the *--skip-package-migration* option: - Example: ``` From 0e96ddb749b948d399e1548f07bdc42409b3644c Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:01:09 +0300 Subject: [PATCH 45/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 7ee47fe34f..5e492da2c1 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8153,7 +8153,7 @@ This command will access repository for corresponding package and retrieve a lis ``` Usage: sonic-package-manager show package versions [OPTIONS] NAME - Print available versions + Show available versions Options: --all Show all available tags in repository From 9090a476b11cf156800bfc27bca32cb9389e4127 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:01:29 +0300 Subject: [PATCH 46/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 5e492da2c1..559774562d 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8183,7 +8183,7 @@ This command will access repository for corresponding package and retrieve a lis **sonic-package-manager show package changelog** -This command fetches the changelog from package manifest and displays it. *NOTE*: package changelog can be retrieved from registry or read from image tarball without installing it. +This command fetches the changelog from the package manifest and displays it. *NOTE*: package changelog can be retrieved from registry or read from image tarball without installing it. - Usage: ``` From b2eeb594534342282b897909fb5ca47cecd30501 Mon Sep 17 00:00:00 2001 From: Stepan Blyshchak <38952541+stepanblyschak@users.noreply.github.com> Date: Wed, 21 Apr 2021 04:04:30 +0300 Subject: [PATCH 47/60] Update doc/Command-Reference.md Co-authored-by: Jan Klare --- doc/Command-Reference.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 559774562d..78be0b0488 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8147,7 +8147,7 @@ This comamnd resets the package by reinstalling it to its default version. *NOTE **sonic-package-manager show package versions** -This command will access repository for corresponding package and retrieve a list of available versions. +This command will retrieve a list of all available versions for the given package from the configured upstream repository - Usage: ``` From e765d5d19bd842f2c0abdb83f5e94cac8108ee3e Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Wed, 21 Apr 2021 04:05:45 +0300 Subject: [PATCH 48/60] review comments Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 3 +-- sonic_package_manager/main.py | 12 +++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index b47b908771..4e4bbc455e 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8046,8 +8046,7 @@ This command will remove a repository as source for SONiC packages from the data **sonic-package-manager install** -This command pulls and installs or upgrades (if already installed) package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run. -The procedure of upgrading a package will restart the corresponding service. +This command pulls and installs a package on SONiC host. *NOTE*: this command requires elevated (root) privileges to run - Usage: ``` diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 1f2d5011d4..d29147f809 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -208,7 +208,7 @@ def manifest(ctx, package_expr, from_repository, from_tarball): - """ Print package manifest content """ + """ Show package manifest """ manager: PackageManager = ctx.obj @@ -228,7 +228,7 @@ def manifest(ctx, @click.option('--plain', is_flag=True, help='Plain output') @click.pass_context def versions(ctx, name, all, plain): - """ Print available versions """ + """ Show available versions """ try: manager: PackageManager = ctx.obj @@ -248,7 +248,7 @@ def changelog(ctx, package_expr, from_repository, from_tarball): - """ Print package changelog """ + """ Show package changelog """ manager: PackageManager = ctx.obj @@ -286,9 +286,7 @@ def changelog(ctx, @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): - """ Add a new repository to database. - Repository in Docker Registry V2. - """ + """ Add a new repository to database """ manager: PackageManager = ctx.obj @@ -306,7 +304,7 @@ def add(ctx, name, repository, default_reference, description): @click.pass_context @root_privileges_required def remove(ctx, name): - """ Remove package from database. """ + """ Remove repository from database. """ manager: PackageManager = ctx.obj From fd46c34536d99524fb701206012f280431533810 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 22 Apr 2021 12:36:47 +0300 Subject: [PATCH 49/60] fix upgrade rollback Signed-off-by: Stepan Blyschak --- sonic_package_manager/manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 2a6ea96b9c..527869ac73 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -521,7 +521,7 @@ def upgrade_from_source(self, if self.feature_registry.is_feature_enabled(old_feature): self._systemctl_action(old_package, 'stop') exits.callback(rollback(self._systemctl_action, - old_package, 'start')) + old_package, 'start')) self.service_creator.remove(old_package, deregister_feature=False) exits.callback(rollback(self.service_creator.create, @@ -539,6 +539,8 @@ def upgrade_from_source(self, if self.feature_registry.is_feature_enabled(new_feature): self._systemctl_action(new_package, 'start') + exits.callback(rollback(self._systemctl_action, + new_package, 'stop')) if not skip_host_plugins: self._install_cli_plugins(new_package) From 86bf53f4b7a79383ecd9921372a568d18277a9b9 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 23 Apr 2021 14:29:07 +0300 Subject: [PATCH 50/60] address review comments Signed-off-by: Stepan Blyschak --- doc/Command-Reference.md | 113 +++++++++++++++++++------------ sonic_installer/main.py | 10 +-- sonic_package_manager/main.py | 78 ++++++++++++--------- sonic_package_manager/manager.py | 6 +- 4 files changed, 122 insertions(+), 85 deletions(-) diff --git a/doc/Command-Reference.md b/doc/Command-Reference.md index 8c7fe7bad2..ab4e28dbdc 100644 --- a/doc/Command-Reference.md +++ b/doc/Command-Reference.md @@ -8013,11 +8013,14 @@ This command will add a new repository as source for SONiC packages to the datab ``` Usage: sonic-package-manager repository add [OPTIONS] NAME REPOSITORY - Add new repository to database + Add a new repository to database. + + NOTE: This command requires elevated (root) privileges to run. Options: - --default-reference TEXT Default installation reference - --description TEXT Optional package entry description + --default-reference TEXT Default installation reference. Can be a tag or + sha256 digest in repository. + --description TEXT Optional package entry description. --help Show this message and exit. ``` - Example: @@ -8034,7 +8037,9 @@ This command will remove a repository as source for SONiC packages from the data ``` Usage: sonic-package-manager repository remove [OPTIONS] NAME - Remove repository from database + Remove repository from database. + + NOTE: This command requires elevated (root) privileges to run. Options: --help Show this message and exit. @@ -8053,32 +8058,46 @@ This command pulls and installs a package on SONiC host. *NOTE*: this command re Usage: sonic-package-manager install [OPTIONS] [PACKAGE_EXPR] Install/Upgrade package using [PACKAGE_EXPR] in format - "[=|@]" + "[=|@]". + + The repository to pull the package from is resolved by lookup in + package database, thus the package has to be added via "sonic- + package-manager repository add" command. + + In case when [PACKAGE_EXPR] is a package name "" this command + will install or upgrade to a version referenced by "default- + reference" in package database. + + NOTE: This command requires elevated (root) privileges to run. Options: - --enable Set the default state of the feature to - enabled and enable feature right after - installation. NOTE: user needs to execute - "config save -y" to make this setting - persistent - --default-owner [local|kube] Default owner configuration setting for a - feature - --from-repository TEXT Fetch package directly from image registry - repository NOTE: This argument is mutually - exclusive with arguments: [package_expr, - from_tarball]. - --from-tarball FILE Fetch package from saved image tarball NOTE: - This argument is mutually exclusive with - arguments: [package_expr, from_repository]. - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG - --skip-host-plugins Do not install host OS plugins provided by the - package (CLI, etc). NOTE: In case when package - host OS plugins are set as mandatory in - package manifest this option will fail the - installation. - --help Show this message and exit. + --enable Set the default state of the feature to enabled + and enable feature right after installation. NOTE: + user needs to execute "config save -y" to make + this setting persistent. + --set-owner [local|kube] Default owner configuration setting for a feature. + --from-repository TEXT Fetch package directly from image registry + repository. NOTE: This argument is mutually + exclusive with arguments: [package_expr, + from_tarball]. + --from-tarball FILE Fetch package from saved image tarball. NOTE: This + argument is mutually exclusive with arguments: + [package_expr, from_repository]. + -f, --force Force operation by ignoring package dependency + tree and package manifest validation failures. + -y, --yes Automatically answer yes on prompts. + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG. + Default is INFO. + --skip-host-plugins Do not install host OS plugins provided by the + package (CLI, etc). NOTE: In case when package + host OS plugins are set as mandatory in package + manifest this option will fail the installation. + --allow-downgrade Allow package downgrade. By default an attempt to + downgrade the package will result in a failure + since downgrade might not be supported by the + package, thus requires explicit request from the + user. + --help Show this message and exit.. ``` - Example: ``` @@ -8099,20 +8118,24 @@ This command pulls and installs a package on SONiC host. *NOTE*: this command re **sonic-package-manager uninstall** -This command uninstalls package from SONiC host. *NOTE*: this command requires elevated (root) privileges to run. +This command uninstalls package from SONiC host. User needs to stop the feature prior to uninstalling it. +*NOTE*: this command requires elevated (root) privileges to run. - Usage: ``` Usage: sonic-package-manager uninstall [OPTIONS] NAME - Uninstall package + Uninstall package. + + NOTE: This command requires elevated (root) privileges to run. Options: - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + -f, --force Force operation by ignoring package dependency tree and + package manifest validation failures. + -y, --yes Automatically answer yes on prompts. + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG. Default + is INFO. --help Show this message and exit. - ``` - Example: ``` @@ -8127,12 +8150,16 @@ This comamnd resets the package by reinstalling it to its default version. *NOTE ``` Usage: sonic-package-manager reset [OPTIONS] NAME - Reset package to the default version + Reset package to the default version. + + NOTE: This command requires elevated (root) privileges to run. Options: - -f, --force Force operation by ignoring failures - -y, --yes Automatically answer yes on prompts - -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG + -f, --force Force operation by ignoring package dependency tree and + package manifest validation failures. + -y, --yes Automatically answer yes on prompts. + -v, --verbosity LVL Either CRITICAL, ERROR, WARNING, INFO or DEBUG. Default + is INFO. --skip-host-plugins Do not install host OS plugins provided by the package (CLI, etc). NOTE: In case when package host OS plugins are set as mandatory in package manifest this option @@ -8152,11 +8179,11 @@ This command will retrieve a list of all available versions for the given packag ``` Usage: sonic-package-manager show package versions [OPTIONS] NAME - Show available versions + Show available versions. Options: - --all Show all available tags in repository - --plain Plain output + --all Show all available tags in repository. + --plain Plain output. --help Show this message and exit. ``` - Example: @@ -8188,7 +8215,7 @@ This command fetches the changelog from the package manifest and displays it. *N ``` Usage: sonic-package-manager show package changelog [OPTIONS] [PACKAGE_EXPR] - Show package changelog + Show package changelog. Options: --from-repository TEXT Fetch package directly from image registry @@ -8217,7 +8244,7 @@ This command fetches the package manifest and displays it. *NOTE*: package manif ``` Usage: sonic-package-manager show package manifest [OPTIONS] [PACKAGE_EXPR] - Show package manifest + Show package manifest. Options: --from-repository TEXT Fetch package directly from image registry diff --git a/sonic_installer/main.py b/sonic_installer/main.py index bc4e7dd338..12a2ab7e0e 100644 --- a/sonic_installer/main.py +++ b/sonic_installer/main.py @@ -233,14 +233,14 @@ def mount_squash_fs(squashfs_path, mount_point): def umount(mount_point, read_only=True, recursive=False, force=True, remove_dir=True): - flags = "-" + flags = [] if read_only: - flags = flags + "r" + flags.append("-r") if force: - flags = flags + "f" + flags.append("-f") if recursive: - flags = flags + "R" - run_command_or_raise(["umount", flags, mount_point]) + flags.append("-R") + run_command_or_raise(["umount", *flags, mount_point]) if remove_dir: run_command_or_raise(["rm", "-rf", mount_point]) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index d29147f809..70e1e67310 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -46,6 +46,8 @@ def wrapped_function(*args, **kwargs): return func(*args, **kwargs) + wrapped_function.__doc__ += '\n\n NOTE: This command requires elevated (root) privileges to run.' + return wrapped_function @@ -91,7 +93,7 @@ def handle_parse_result(self, ctx, opts, args): PACKAGE_SOURCE_OPTIONS = [ click.option('--from-repository', - help='Fetch package directly from image registry repository', + help='Fetch package directly from image registry repository.', cls=MutuallyExclusiveOption, mutually_exclusive=['from_tarball', 'package_expr']), click.option('--from-tarball', @@ -99,7 +101,7 @@ def handle_parse_result(self, ctx, opts, args): readable=True, file_okay=True, dir_okay=False), - help='Fetch package from saved image tarball', + help='Fetch package from saved image tarball.', cls=MutuallyExclusiveOption, mutually_exclusive=['from_repository', 'package_expr']), click.argument('package-expr', @@ -110,21 +112,21 @@ def handle_parse_result(self, ctx, opts, args): PACKAGE_COMMON_INSTALL_OPTIONS = [ click.option('--skip-host-plugins', - is_flag=True, - help='Do not install host OS plugins provided by the package (CLI, etc). ' - 'NOTE: In case when package host OS plugins are set as mandatory in ' - 'package manifest this option will fail the installation.'), + is_flag=True, + help='Do not install host OS plugins provided by the package (CLI, etc). ' + 'NOTE: In case when package host OS plugins are set as mandatory in ' + 'package manifest this option will fail the installation.') ] PACKAGE_COMMON_OPERATION_OPTIONS = [ click.option('-f', '--force', is_flag=True, - help='Force operation by ignoring failures'), + help='Force operation by ignoring package dependency tree and package manifest validation failures.'), click.option('-y', '--yes', is_flag=True, - help='Automatically answer yes on prompts'), - click_log.simple_verbosity_option(log), + help='Automatically answer yes on prompts.'), + click_log.simple_verbosity_option(log, help='Either CRITICAL, ERROR, WARNING, INFO or DEBUG. Default is INFO.'), ] @@ -150,7 +152,7 @@ def cli(ctx): @cli.group() @click.pass_context def repository(ctx): - """ Repository management commands """ + """ Repository management commands. """ pass @@ -158,7 +160,7 @@ def repository(ctx): @cli.group() @click.pass_context def show(ctx): - """ Package manager show commands """ + """ Package manager show commands. """ pass @@ -166,7 +168,7 @@ def show(ctx): @show.group() @click.pass_context def package(ctx): - """ Package show commands """ + """ Package show commands. """ pass @@ -174,7 +176,7 @@ def package(ctx): @cli.command() @click.pass_context def list(ctx): - """ List available packages """ + """ List available packages. """ table_header = ['Name', 'Repository', 'Description', 'Version', 'Status'] table_body = [] @@ -208,7 +210,7 @@ def manifest(ctx, package_expr, from_repository, from_tarball): - """ Show package manifest """ + """ Show package manifest. """ manager: PackageManager = ctx.obj @@ -224,11 +226,11 @@ def manifest(ctx, @package.command() @click.argument('name') -@click.option('--all', is_flag=True, help='Show all available tags in repository') -@click.option('--plain', is_flag=True, help='Plain output') +@click.option('--all', is_flag=True, help='Show all available tags in repository.') +@click.option('--plain', is_flag=True, help='Plain output.') @click.pass_context def versions(ctx, name, all, plain): - """ Show available versions """ + """ Show available versions. """ try: manager: PackageManager = ctx.obj @@ -248,7 +250,7 @@ def changelog(ctx, package_expr, from_repository, from_tarball): - """ Show package changelog """ + """ Show package changelog. """ manager: PackageManager = ctx.obj @@ -281,12 +283,12 @@ def changelog(ctx, @repository.command() @click.argument('name', type=str) @click.argument('repository', type=str) -@click.option('--default-reference', type=str, help='Default installation reference') -@click.option('--description', type=str, help='Optional package entry description') +@click.option('--default-reference', type=str, help='Default installation reference. Can be a tag or sha256 digest in repository.') +@click.option('--description', type=str, help='Optional package entry description.') @click.pass_context @root_privileges_required def add(ctx, name, repository, default_reference, description): - """ Add a new repository to database """ + """ Add a new repository to database. """ manager: PackageManager = ctx.obj @@ -320,11 +322,16 @@ def remove(ctx, name): help='Set the default state of the feature to enabled ' 'and enable feature right after installation. ' 'NOTE: user needs to execute "config save -y" to make ' - 'this setting persistent') -@click.option('--default-owner', + 'this setting persistent.') +@click.option('--set-owner', type=click.Choice(['local', 'kube']), default='local', - help='Default owner configuration setting for a feature') + help='Default owner configuration setting for a feature.') +@click.option('--allow-downgrade', + is_flag=True, + help='Allow package downgrade. By default an attempt to downgrade the package ' + 'will result in a failure since downgrade might not be supported by the package, ' + 'thus requires explicit request from the user.') @add_options(PACKAGE_SOURCE_OPTIONS) @add_options(PACKAGE_COMMON_OPERATION_OPTIONS) @add_options(PACKAGE_COMMON_INSTALL_OPTIONS) @@ -337,10 +344,16 @@ def install(ctx, force, yes, enable, - default_owner, - skip_host_plugins): - """ Install/Upgrade package using [PACKAGE_EXPR] in format - "[=|@]" """ + set_owner, + skip_host_plugins, + allow_downgrade): + """ Install/Upgrade package using [PACKAGE_EXPR] in format "[=|@]". + + The repository to pull the package from is resolved by lookup in package database, + thus the package has to be added via "sonic-package-manager repository add" command. + + In case when [PACKAGE_EXPR] is a package name "" this command will install or upgrade + to a version referenced by "default-reference" in package database. """ manager: PackageManager = ctx.obj @@ -355,8 +368,9 @@ def install(ctx, install_opts = { 'force': force, 'enable': enable, - 'default_owner': default_owner, + 'default_owner': set_owner, 'skip_host_plugins': skip_host_plugins, + 'allow_downgrade': allow_downgrade, } try: @@ -377,7 +391,7 @@ def install(ctx, @click.pass_context @root_privileges_required def reset(ctx, name, force, yes, skip_host_plugins): - """ Reset package to the default version """ + """ Reset package to the default version. """ manager: PackageManager = ctx.obj @@ -399,7 +413,7 @@ def reset(ctx, name, force, yes, skip_host_plugins): @click.pass_context @root_privileges_required def uninstall(ctx, name, force, yes): - """ Uninstall package """ + """ Uninstall package. """ manager: PackageManager = ctx.obj @@ -422,7 +436,7 @@ def uninstall(ctx, name, force, yes): @click.pass_context @root_privileges_required def migrate(ctx, database, force, yes, dockerd_socket): - """ Migrate packages from the given database file """ + """ Migrate packages from the given database file. """ manager: PackageManager = ctx.obj diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 527869ac73..fb15c91a7c 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -91,11 +91,7 @@ def opt_check(func: Callable) -> Callable: @functools.wraps(func) def wrapped_function(*args, **kwargs): sig = signature(func) - redundant_opts = [opt for opt in kwargs if opt not in sig.parameters] - if redundant_opts: - raise PackageManagerError( - f'Unsupported options: {",".join(redundant_opts)} for {func.__name__}' - ) + kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters} return func(*args, **kwargs) return wrapped_function From 1ace0e121b8b2367273d700056ecbe2349e6dfed Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 26 Apr 2021 13:11:09 +0300 Subject: [PATCH 51/60] fix command line options for install/upgrade Signed-off-by: Stepan Blyschak --- sonic_package_manager/main.py | 13 +++++++++---- sonic_package_manager/manager.py | 6 +++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/sonic_package_manager/main.py b/sonic_package_manager/main.py index 70e1e67310..c0589ae5b5 100644 --- a/sonic_package_manager/main.py +++ b/sonic_package_manager/main.py @@ -319,16 +319,18 @@ def remove(ctx, name): @cli.command() @click.option('--enable', is_flag=True, + default=None, help='Set the default state of the feature to enabled ' 'and enable feature right after installation. ' 'NOTE: user needs to execute "config save -y" to make ' 'this setting persistent.') @click.option('--set-owner', type=click.Choice(['local', 'kube']), - default='local', + default=None, help='Default owner configuration setting for a feature.') @click.option('--allow-downgrade', is_flag=True, + default=None, help='Allow package downgrade. By default an attempt to downgrade the package ' 'will result in a failure since downgrade might not be supported by the package, ' 'thus requires explicit request from the user.') @@ -367,11 +369,14 @@ def install(ctx, install_opts = { 'force': force, - 'enable': enable, - 'default_owner': set_owner, 'skip_host_plugins': skip_host_plugins, - 'allow_downgrade': allow_downgrade, } + if enable is not None: + install_opts['enable'] = enable + if set_owner is not None: + install_opts['default_owner'] = set_owner + if allow_downgrade is not None: + install_opts['allow_downgrade'] = allow_downgrade try: manager.install(package_expr, diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index fb15c91a7c..9f3f5d95d4 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -91,7 +91,11 @@ def opt_check(func: Callable) -> Callable: @functools.wraps(func) def wrapped_function(*args, **kwargs): sig = signature(func) - kwargs = {k: v for k, v in kwargs.items() if k in sig.parameters} + unsupported_opts = [opt for opt in kwargs if opt not in sig.parameters] + if unsupported_opts: + raise PackageManagerError( + f'Unsupported options {unsupported_opts} for {func.__name__}' + ) return func(*args, **kwargs) return wrapped_function From 6feda3645d9720d0c5a3499b02b97e8c73720591 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Mon, 26 Apr 2021 17:15:50 +0300 Subject: [PATCH 52/60] fix bug when have components constraints Signed-off-by: Stepan Blyschak --- sonic_package_manager/constraint.py | 2 +- sonic_package_manager/service_creator/creator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sonic_package_manager/constraint.py b/sonic_package_manager/constraint.py index 8b044f3ec9..af5a13000b 100644 --- a/sonic_package_manager/constraint.py +++ b/sonic_package_manager/constraint.py @@ -58,7 +58,7 @@ def deparse(self) -> Dict[str, str]: """ return { - component: str(version) for component, version in self.components + component: str(version) for component, version in self.components.items() } diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index d86b70f979..2b305b04b3 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -123,7 +123,7 @@ def create(self, self.feature_registry.register(package.manifest, state, owner) except (Exception, KeyboardInterrupt): - self.remove(package, not register_feature) + self.remove(package, register_feature) raise def remove(self, package: Package, deregister_feature=True): From 0d5dfcfc0570d69184a159045aa6c25cd0854da6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 29 Apr 2021 00:30:47 +0300 Subject: [PATCH 53/60] Fix review comments Signed-off-by: Stepan Blyschak --- setup.py | 1 - sonic_installer/bootloader/bootloader.py | 2 +- sonic_package_manager/manager.py | 1 + sonic_package_manager/manifest.py | 1 + sonic_package_manager/registry.py | 1 + sonic_package_manager/service_creator/creator.py | 1 + sonic_package_manager/service_creator/sonic_db.py | 1 + tests/sonic_package_manager/test_constraint.py | 1 + tests/sonic_package_manager/test_manager.py | 1 + tests/sonic_package_manager/test_metadata.py | 1 + tests/sonic_package_manager/test_reference.py | 1 + tests/sonic_package_manager/test_service_creator.py | 1 + 12 files changed, 11 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a5bbc36dfa..4cbdf52ff9 100644 --- a/setup.py +++ b/setup.py @@ -191,7 +191,6 @@ tests_require = [ 'pyfakefs', 'pytest', - 'mock>=2.0.0', 'mockredispy>=2.9.3', 'sonic-config-engine', 'deepdiff==5.2.3' diff --git a/sonic_installer/bootloader/bootloader.py b/sonic_installer/bootloader/bootloader.py index 607cd730db..a6694977ae 100644 --- a/sonic_installer/bootloader/bootloader.py +++ b/sonic_installer/bootloader/bootloader.py @@ -72,4 +72,4 @@ def get_image_path(cls, image): @contextmanager def get_path_in_image(self, image_path, path_in_image): """returns the path to the squashfs""" - yield path.join(image_path, path_in_image) \ No newline at end of file + yield path.join(image_path, path_in_image) diff --git a/sonic_package_manager/manager.py b/sonic_package_manager/manager.py index 9f3f5d95d4..ba437534ed 100644 --- a/sonic_package_manager/manager.py +++ b/sonic_package_manager/manager.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import contextlib import functools import os diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index 1bd5449d3a..b58a0d10f0 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + from abc import ABC from dataclasses import dataclass from typing import Optional, List, Dict, Any diff --git a/sonic_package_manager/registry.py b/sonic_package_manager/registry.py index bf4308efa0..8a09d9136e 100644 --- a/sonic_package_manager/registry.py +++ b/sonic_package_manager/registry.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import json from dataclasses import dataclass from typing import List, Dict diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 2b305b04b3..54b9315bee 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import contextlib import os import stat diff --git a/sonic_package_manager/service_creator/sonic_db.py b/sonic_package_manager/service_creator/sonic_db.py index a9ba837ab1..a064c60c4a 100644 --- a/sonic_package_manager/service_creator/sonic_db.py +++ b/sonic_package_manager/service_creator/sonic_db.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import contextlib import json import os diff --git a/tests/sonic_package_manager/test_constraint.py b/tests/sonic_package_manager/test_constraint.py index 2e7067ef63..1b34a301d2 100644 --- a/tests/sonic_package_manager/test_constraint.py +++ b/tests/sonic_package_manager/test_constraint.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + from sonic_package_manager import version from sonic_package_manager.constraint import PackageConstraint from sonic_package_manager.version import Version, VersionRange diff --git a/tests/sonic_package_manager/test_manager.py b/tests/sonic_package_manager/test_manager.py index dc79f7f483..c7eb1ca7ac 100644 --- a/tests/sonic_package_manager/test_manager.py +++ b/tests/sonic_package_manager/test_manager.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + from unittest.mock import Mock, call import pytest diff --git a/tests/sonic_package_manager/test_metadata.py b/tests/sonic_package_manager/test_metadata.py index 4636c18282..aee2f49428 100644 --- a/tests/sonic_package_manager/test_metadata.py +++ b/tests/sonic_package_manager/test_metadata.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import contextlib from unittest.mock import Mock, MagicMock diff --git a/tests/sonic_package_manager/test_reference.py b/tests/sonic_package_manager/test_reference.py index c986632c43..043b66ddd5 100644 --- a/tests/sonic_package_manager/test_reference.py +++ b/tests/sonic_package_manager/test_reference.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import pytest from sonic_package_manager.reference import PackageReference diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index c540259b49..fec8de600c 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -1,4 +1,5 @@ #!/usr/bin/env python + import os from unittest.mock import Mock, MagicMock From e91ff811a339f8a9aef285d751620849ace7257d Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 04:42:29 +0300 Subject: [PATCH 54/60] fix script Signed-off-by: Stepan Blyschak --- scripts/generate_shutdown_order.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_shutdown_order.py b/scripts/generate_shutdown_order.py index db9f48d676..a9a1168a05 100644 --- a/scripts/generate_shutdown_order.py +++ b/scripts/generate_shutdown_order.py @@ -6,7 +6,7 @@ def main(): manager = PackageManager.get_manager() - installed_packages = manager.get_installed_packages_list() + installed_packages = manager.get_installed_packages() print('installed packages {}'.format(installed_packages)) manager.service_creator.generate_shutdown_sequence_files(installed_packages) print('Done.') From 62e26463ae4bfc333addd792b6c3738da99de755 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 04:52:05 +0300 Subject: [PATCH 55/60] fix unresolved conftest.py Signed-off-by: Stepan Blyschak --- tests/sonic_package_manager/conftest.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index 236376bb6b..9d5fc01b59 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -75,7 +75,6 @@ def __init__(self): components={ 'libswsscommon': Version.parse('1.0.0'), 'libsairedis': Version.parse('1.0.0') -<<<<<<< HEAD }, warm_shutdown={ 'before': ['syncd'], @@ -106,8 +105,6 @@ def __init__(self): }, fast_shutdown={ 'before': ['swss'], -======= ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e } ) self.add('Azure/docker-test', '1.6.0', 'test-package', '1.6.0') @@ -141,13 +138,9 @@ def from_tarball(self, filepath: str) -> Manifest: components = self.metadata_store[path][ref]['components'] return Metadata(manifest, components) -<<<<<<< HEAD def add(self, repo, reference, name, version, components=None, warm_shutdown=None, fast_shutdown=None, processes=None): -======= - def add(self, repo, reference, name, version, components=None): ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e repo_dict = self.metadata_store.setdefault(repo, {}) repo_dict[reference] = { 'manifest': { @@ -158,14 +151,10 @@ def add(self, repo, reference, name, version, components=None): }, 'service': { 'name': name, -<<<<<<< HEAD 'warm-shutdown': warm_shutdown or {}, 'fast-shutdown': fast_shutdown or {}, }, 'processes': processes or {} -======= - } ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e }, 'components': components or {}, } @@ -238,7 +227,6 @@ def fake_db(fake_metadata_resolver): add_package( content, fake_metadata_resolver, -<<<<<<< HEAD 'docker-syncd', 'latest', description='SONiC syncd service', @@ -259,8 +247,6 @@ def fake_db(fake_metadata_resolver): add_package( content, fake_metadata_resolver, -======= ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e 'Azure/docker-test', '1.6.0', description='SONiC Package Manager Test Package', From 79cabb201051e76cb383b7cb3a5751e619127b6b Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 04:52:36 +0300 Subject: [PATCH 56/60] fix unresolved test_service_creator.py Signed-off-by: Stepan Blyschak --- .../test_service_creator.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/sonic_package_manager/test_service_creator.py b/tests/sonic_package_manager/test_service_creator.py index 1c05b98d55..ffa6737531 100644 --- a/tests/sonic_package_manager/test_service_creator.py +++ b/tests/sonic_package_manager/test_service_creator.py @@ -28,7 +28,6 @@ def manifest(): 'dependent-of': ['swss'], 'asic-service': False, 'host-service': True, -<<<<<<< HEAD 'warm-shutdown': { 'before': ['syncd'], 'after': ['swss'], @@ -36,15 +35,12 @@ def manifest(): 'fast-shutdown': { 'before': ['swss'], }, -======= ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e }, 'container': { 'privileged': True, 'volumes': [ '/etc/sonic:/etc/sonic:ro' ] -<<<<<<< HEAD }, 'processes': [ { @@ -70,24 +66,12 @@ def test_service_creator(sonic_fs, manifest, package_manager, mock_feature_regis installed_packages = package_manager._get_installed_packages_and(package) creator.create(package) creator.generate_shutdown_sequence_files(installed_packages) -======= - } - }) - - -def test_service_creator(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): - creator = ServiceCreator(mock_feature_registry, mock_sonic_db) - entry = PackageEntry('test', 'azure/sonic-test') - package = Package(entry, Metadata(manifest)) - creator.create(package) ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) assert sonic_fs.exists(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, 'test.sh')) assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) -<<<<<<< HEAD def read_file(name): with open(os.path.join(ETC_SONIC_PATH, name)) as file: return file.read() @@ -96,8 +80,6 @@ def read_file(name): assert read_file('fast-reboot_order') == 'teamd test swss syncd' assert read_file('test_reconcile') == 'test-process test-process-3' -======= ->>>>>>> 08337aa7637b290bb8407c38b2a5dbe3e8383b3e def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry, mock_sonic_db): creator = ServiceCreator(mock_feature_registry, mock_sonic_db) From a23fe442ba09471d2192d2f57ac02023453fe8c2 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 05:49:13 +0300 Subject: [PATCH 57/60] fix lgtm warning Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 125425b377..8987554ea6 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -5,7 +5,7 @@ import stat import subprocess from collections import defaultdict -from typing import Dict, List, Optional +from typing import Dict, Optional import jinja2 as jinja2 from prettyprinter import pformat From f4be57955da555e8650b5e2e56ed10a31bc99476 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Fri, 30 Apr 2021 06:30:53 +0300 Subject: [PATCH 58/60] fix lgtm warning Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 8987554ea6..9b4081a82d 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -5,7 +5,7 @@ import stat import subprocess from collections import defaultdict -from typing import Dict, Optional +from typing import Dict import jinja2 as jinja2 from prettyprinter import pformat From 0ba3cf40e3bbd7bb8ad36fdedfb5be2567631dc0 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 13 May 2021 13:48:11 +0300 Subject: [PATCH 59/60] improve manifest types validation Signed-off-by: Stepan Blyschak --- sonic_package_manager/manifest.py | 16 ++++++++++------ tests/sonic_package_manager/conftest.py | 2 +- tests/sonic_package_manager/test_manifest.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/sonic_package_manager/manifest.py b/sonic_package_manager/manifest.py index d2b9dab818..c126e2eef1 100644 --- a/sonic_package_manager/manifest.py +++ b/sonic_package_manager/manifest.py @@ -92,8 +92,10 @@ class ManifestRoot(ManifestNode): def marshal(self, value: Optional[dict]): result = {} - if value is None: - value = {} + value = value or {} + + if not isinstance(value, dict): + raise ManifestError(f'"{self.key}" field has to be a dictionary') for item in self.items: next_value = value.get(item.key) @@ -115,7 +117,7 @@ def marshal(self, value): if value is None: if self.default is not None: return self.default - raise ManifestError(f'{self.key} is a required field but it is missing') + raise ManifestError(f'"{self.key}" is a required field but it is missing') try: return_value = self.type.marshal(value) except Exception as err: @@ -130,10 +132,12 @@ class ManifestArray(ManifestNode): type: Any def marshal(self, value): - if value is None: - return [] - return_value = [] + value = value or [] + + if not isinstance(value, list): + raise ManifestError(f'"{self.key}" has to be of type list') + try: for item in value: return_value.append(self.type.marshal(item)) diff --git a/tests/sonic_package_manager/conftest.py b/tests/sonic_package_manager/conftest.py index 9d5fc01b59..2788a75cd3 100644 --- a/tests/sonic_package_manager/conftest.py +++ b/tests/sonic_package_manager/conftest.py @@ -154,7 +154,7 @@ def add(self, repo, reference, name, version, components=None, 'warm-shutdown': warm_shutdown or {}, 'fast-shutdown': fast_shutdown or {}, }, - 'processes': processes or {} + 'processes': processes or [], }, 'components': components or {}, } diff --git a/tests/sonic_package_manager/test_manifest.py b/tests/sonic_package_manager/test_manifest.py index efdcc558ab..2f201b8107 100644 --- a/tests/sonic_package_manager/test_manifest.py +++ b/tests/sonic_package_manager/test_manifest.py @@ -57,6 +57,20 @@ def test_manifest_v1_mounts_invalid(): 'mounts': [{'not-source': 'a', 'target': 'b', 'type': 'bind'}]}}) +def test_manifest_invalid_root_type(): + manifest_json_input = {'package': { 'name': 'test', 'version': '1.0.0'}, + 'service': {'name': 'test'}, 'container': 'abc'} + with pytest.raises(ManifestError): + Manifest.marshal(manifest_json_input) + + +def test_manifest_invalid_array_type(): + manifest_json_input = {'package': { 'name': 'test', 'version': '1.0.0'}, + 'service': {'name': 'test', 'warm-shutdown': {'after': 'bgp'}}} + with pytest.raises(ManifestError): + Manifest.marshal(manifest_json_input) + + def test_manifest_v1_unmarshal(): manifest_json_input = {'package': {'name': 'test', 'version': '1.0.0', 'depends': [ From 1554f001fcc62dcbd5b6e148dd77b4b14a43a2e6 Mon Sep 17 00:00:00 2001 From: Stepan Blyschak Date: Thu, 13 May 2021 16:49:05 +0300 Subject: [PATCH 60/60] fix docstrings Signed-off-by: Stepan Blyschak --- sonic_package_manager/service_creator/creator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sonic_package_manager/service_creator/creator.py b/sonic_package_manager/service_creator/creator.py index 9b4081a82d..4c618eb7ea 100644 --- a/sonic_package_manager/service_creator/creator.py +++ b/sonic_package_manager/service_creator/creator.py @@ -440,7 +440,7 @@ def generate_service_reconciliation_file(self, package): (path: /etc/sonic/_reconcile). Args: - package: Package object to generate dump plugin script for. + package: Package object to generate service reconciliation file for. Returns: None """ @@ -456,7 +456,7 @@ def set_initial_config(self, package): This method updates but does not override existing entries in tables. Args: - package: Package object to generate dump plugin script for. + package: Package object to set initial configuration for. Returns: None """ @@ -485,7 +485,7 @@ def remove_config(self, package): TODO: remove config from tables known to yang model Args: - package: Package object to generate dump plugin script for. + package: Package object remove initial configuration for. Returns: None """