diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index c322c58aa00..d06b37b8da0 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -199,6 +199,16 @@ sudo rm -rf $FILESYSTEM_ROOT/$SONIC_UTILITIES_PY2_WHEEL_NAME sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-utilities-data_*.deb || \ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f +# Install SONiC Package Manager Python 3 package +SONIC_PACKAGE_MANAGER_PY3_WHEEL_NAME=$(basename {{sonic_package_manager_py3_wheel_path}}) +sudo cp {{sonic_package_manager_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_PACKAGE_MANAGER_PY3_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_PACKAGE_MANAGER_PY3_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SONIC_PACKAGE_MANAGER_PY3_WHEEL_NAME + +# Install sonic-package-manager data files (and any dependencies via 'apt-get -y install -f') +sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-package-manager-data_*.deb || \ + sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f + # sonic-utilities-data installs bash-completion as a dependency. However, it is disabled by default # in bash.bashrc, so we copy a version of the file with it enabled here. sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/ diff --git a/rules/sonic-package-manager-data.dep b/rules/sonic-package-manager-data.dep new file mode 100644 index 00000000000..c1f0153fb75 --- /dev/null +++ b/rules/sonic-package-manager-data.dep @@ -0,0 +1,9 @@ +SPATH := $($(SONIC_PACKAGE_MANAGER_DATA)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-package-manager-data.mk rules/sonic-package-manager-data.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +DEP_FILES += $(shell git ls-files $(SPATH)) + +$(SONIC_PACKAGE_MANAGER_DATA)_CACHE_MODE := GIT_CONTENT_SHA +$(SONIC_PACKAGE_MANAGER_DATA)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(SONIC_PACKAGE_MANAGER_DATA)_DEP_FILES := $(DEP_FILES) + diff --git a/rules/sonic-package-manager-data.mk b/rules/sonic-package-manager-data.mk new file mode 100644 index 00000000000..7677dafb038 --- /dev/null +++ b/rules/sonic-package-manager-data.mk @@ -0,0 +1,6 @@ +# SONiC command line utilities data package + +SONIC_PACKAGE_MANAGER_DATA = sonic-package-manager-data_1.0-1_all.deb +$(SONIC_PACKAGE_MANAGER_DATA)_SRC_PATH = $(SRC_PATH)/sonic-package-manager/sonic-package-manager-data +SONIC_DPKG_DEBS += $(SONIC_PACKAGE_MANAGER_DATA) + diff --git a/rules/sonic-package-manager.dep b/rules/sonic-package-manager.dep new file mode 100644 index 00000000000..37f81fa1dd5 --- /dev/null +++ b/rules/sonic-package-manager.dep @@ -0,0 +1,11 @@ +SPATH := $($(SONIC_PACKAGE_MANAGER_PY3)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-package-manager.mk rules/sonic-package-manager.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files)) + +$(SONIC_PACKAGE_MANAGER_PY3)_CACHE_MODE := GIT_CONTENT_SHA +$(SONIC_PACKAGE_MANAGER_PY3)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(SONIC_PACKAGE_MANAGER_PY3)_DEP_FILES := $(DEP_FILES) +$(SONIC_PACKAGE_MANAGER_PY3)_SMDEP_FILES := $(SMDEP_FILES) +$(SONIC_PACKAGE_MANAGER_PY3)_SMDEP_PATHS := $(SPATH) + diff --git a/rules/sonic-package-manager.mk b/rules/sonic-package-manager.mk new file mode 100644 index 00000000000..02909e90c6b --- /dev/null +++ b/rules/sonic-package-manager.mk @@ -0,0 +1,10 @@ +# sonic package manager python package + +SONIC_PACKAGE_MANAGER_PY3 = sonic_package_manager-1.0-py3-none-any.whl +$(SONIC_PACKAGE_MANAGER_PY3)_SRC_PATH = $(SRC_PATH)/sonic-package-manager/ +$(SONIC_PACKAGE_MANAGER_PY3)_PYTHON_VERSION = 3 +$(SONIC_PACKAGE_MANAGER_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3) \ + $(SWSSSDK_PY3) # Temporary dependency till issue with py-swsscommon is resolved +$(SONIC_PACKAGE_MANAGER_PY3)_DEBS_DEPENDS += $(PYTHON3_SWSSCOMMON) +SONIC_PYTHON_WHEELS += $(SONIC_PACKAGE_MANAGER_PY3) + diff --git a/slave.mk b/slave.mk index ba82e717910..e7b8155c4c0 100644 --- a/slave.mk +++ b/slave.mk @@ -806,6 +806,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(PYTHON_SWSSCOMMON) \ $(PYTHON3_SWSSCOMMON) \ $(SONIC_UTILITIES_DATA) \ + $(SONIC_PACKAGE_MANAGER_DATA) \ $(SONIC_HOST_SERVICES_DATA)) \ $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \ $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \ @@ -823,7 +824,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \ - $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3)) + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3)) \ + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PACKAGE_MANAGER_PY3)) $(HEADER) # Pass initramfs and linux kernel explicitly. They are used for all platforms export debs_path="$(IMAGE_DISTRO_DEBS_PATH)" @@ -869,6 +871,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))" export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON3_SWSSCOMMON))" export sonic_utilities_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY2))" + export sonic_package_manager_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PACKAGE_MANAGER_PY3))" export sonic_host_services_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3))" $(foreach docker, $($*_DOCKERS),\ diff --git a/src/sonic-package-manager/.gitignore b/src/sonic-package-manager/.gitignore new file mode 100644 index 00000000000..9822cb118f0 --- /dev/null +++ b/src/sonic-package-manager/.gitignore @@ -0,0 +1,142 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# IDEs +.idea/ +.vscode/ diff --git a/src/sonic-package-manager/pytest.ini b/src/sonic-package-manager/pytest.ini new file mode 100644 index 00000000000..173f1fc3ba1 --- /dev/null +++ b/src/sonic-package-manager/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --cov=sonic_package_manager --cov-report html --cov-report term --cov-report xml diff --git a/src/sonic-package-manager/setup.cfg b/src/sonic-package-manager/setup.cfg new file mode 100644 index 00000000000..b7e478982cc --- /dev/null +++ b/src/sonic-package-manager/setup.cfg @@ -0,0 +1,2 @@ +[aliases] +test=pytest diff --git a/src/sonic-package-manager/setup.py b/src/sonic-package-manager/setup.py new file mode 100644 index 00000000000..c3811dede3b --- /dev/null +++ b/src/sonic-package-manager/setup.py @@ -0,0 +1,58 @@ +from setuptools import setup + +setup( + name='sonic-package-manager', + version='1.0', + description='SONiC Package Manager', + author='Stepan Blyshchak', + author_email='stepanb@nvidia.com', + url='https://github.com/Azure/sonic-buildimage', + packages=[ + 'sonic_package_manager', + 'sonic_package_manager/service_creator' + ], + install_requires = [ + 'jinja2>=2.10', + 'tabulate', + 'poetry-semver', + 'click', + 'click_log', + 'docker', + 'docker-image-py', + 'enlighten', + 'natsort', + 'sonic-py-common', + 'swsssdk', # Temporary dependency till the issue with py-swsscommon is resolved. + 'requests', + ], + setup_requires=[ + 'pytest-runner', + 'wheel', + ], + tests_require=[ + 'pytest', + 'mock>=2.0.0', + 'pytest-sugar', + 'pytest-cov', + 'pyfakefs', + ], + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + classifiers=[ + 'Intended Audience :: Developers', + 'Natural Language :: English', + "Programming Language :: Python :: 2", + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + ], + entry_points={ + 'console_scripts': [ + 'sonic-package-manager = sonic_package_manager.main:cli', + 'spm = sonic_package_manager.main:cli', + ] + }, + test_suite='setup.get_test_suite', +) diff --git a/src/sonic-package-manager/sonic-package-manager-data/bash_completion.d/sonic-package-manager b/src/sonic-package-manager/sonic-package-manager-data/bash_completion.d/sonic-package-manager new file mode 100644 index 00000000000..a8a24566035 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/bash_completion.d/sonic-package-manager @@ -0,0 +1,8 @@ +_sonic_package_manager_completion() { + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) ) + return 0 +} + +complete -F _sonic_package_manager_completion -o default sonic-package-manager; diff --git a/src/sonic-package-manager/sonic-package-manager-data/bash_completion.d/spm b/src/sonic-package-manager/sonic-package-manager-data/bash_completion.d/spm new file mode 100644 index 00000000000..8931dc389ce --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/bash_completion.d/spm @@ -0,0 +1,8 @@ +_sonic_package_manager_completion() { + COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \ + COMP_CWORD=$COMP_CWORD \ + _SONIC_PACKAGE_MANAGER_COMPLETE=complete $1 ) ) + return 0 +} + +complete -F _sonic_package_manager_completion -o default spm; diff --git a/src/sonic-package-manager/sonic-package-manager-data/debian/changelog b/src/sonic-package-manager/sonic-package-manager-data/debian/changelog new file mode 100644 index 00000000000..6ea33fa907b --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/debian/changelog @@ -0,0 +1,5 @@ +sonic-package-manager-data (1.0-1) UNRELEASED; urgency=low + + * Initial release + + -- Stepan Blyshchak Thu, 1 Oct 2020 00:30:00 +0000 diff --git a/src/sonic-package-manager/sonic-package-manager-data/debian/compat b/src/sonic-package-manager/sonic-package-manager-data/debian/compat new file mode 100644 index 00000000000..ec635144f60 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/debian/compat @@ -0,0 +1 @@ +9 diff --git a/src/sonic-package-manager/sonic-package-manager-data/debian/control b/src/sonic-package-manager/sonic-package-manager-data/debian/control new file mode 100644 index 00000000000..7e03e6675a5 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/debian/control @@ -0,0 +1,11 @@ +Source: sonic-package-manager-data +Maintainer: Stepan Blyshchak +Section: misc +Priority: optional +Standards-Version: 0.1 +Build-Depends: debhelper (>=9) + +Package: sonic-package-manager-data +Architecture: all +Depends: ${misc:Depends} bash-completion +Description: Data files required for SONiC Package Manager diff --git a/src/sonic-package-manager/sonic-package-manager-data/debian/copyright b/src/sonic-package-manager/sonic-package-manager-data/debian/copyright new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/sonic-package-manager/sonic-package-manager-data/debian/install b/src/sonic-package-manager/sonic-package-manager-data/debian/install new file mode 100644 index 00000000000..82d087d54d3 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/debian/install @@ -0,0 +1,2 @@ +bash_completion.d/ /etc/ +templates/*.j2 /usr/share/sonic/templates/ diff --git a/src/sonic-package-manager/sonic-package-manager-data/debian/rules b/src/sonic-package-manager/sonic-package-manager-data/debian/rules new file mode 100755 index 00000000000..a5e2f5acfc1 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/debian/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f + +build: + +%: + dh $@ diff --git a/src/sonic-package-manager/sonic-package-manager-data/templates/dump.sh.j2 b/src/sonic-package-manager/sonic-package-manager-data/templates/dump.sh.j2 new file mode 100644 index 00000000000..bee86ab87d7 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/templates/dump.sh.j2 @@ -0,0 +1,10 @@ +#!/bin/bash + +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# + +DEV="$1" + +docker exec -it {{ manifest.service.name }}$DEV {{ manifest.package['debug-dump'] }} diff --git a/src/sonic-package-manager/sonic-package-manager-data/templates/monit.conf.j2 b/src/sonic-package-manager/sonic-package-manager-data/templates/monit.conf.j2 new file mode 100644 index 00000000000..f51efb9beea --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/templates/monit.conf.j2 @@ -0,0 +1,18 @@ +############################################################################### +## +## =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +## auto-generated from {{ source }} by sonic-package-manager +## +## Monit configuration for {{ feature }} service +## process list: +{%- for process in processes %} +{%- if process.critical %} +## {{ process.name }} +{%- endif %} +{%- endfor %} +############################################################################### +{%- for process in processes %} +check program {{ feature }}|{{ process.name }} with path "/usr/bin/process_checker {{ feature }} {{ process.command }}" + if status != 0 for 5 times within 5 cycles then alert + +{% endfor %} diff --git a/src/sonic-package-manager/sonic-package-manager-data/templates/service_mgmt.sh.j2 b/src/sonic-package-manager/sonic-package-manager-data/templates/service_mgmt.sh.j2 new file mode 100644 index 00000000000..86632904a80 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/templates/service_mgmt.sh.j2 @@ -0,0 +1,149 @@ +#!/bin/bash + +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# + +SERVICE="{{ manifest.package.name }}" +NAMESPACE_PREFIX="asic" +SONIC_DB_CLI="sonic-db-cli" +TMPDIR="/tmp/" +DEBUGLOG="${TMPDIR}/${SERVICE}.log" +[[ ! -z $DEV ]] && DEBUGLOG="${TMPDIR}/${SERVICE}-${DEV}.log" +[[ ! -z $DEV ]] && NET_NS="${NAMESPACE_PREFIX}${DEV}" # name of the network namespace +[[ ! -z $DEV ]] && SONIC_DB_CLI="${SONIC_DB_CLI} -n ${NET_NS}" + +{%- for service in manifest.service.dependent %} +{%- if service in multi_instance_services %} +MULTI_INST_DEPENDENT="${MULTI_INST_DEPENDENT} {{ service }}" +{%- else %} +DEPENDENT="${DEPENDENT} {{ service }}" +{%- endif %} +{%- endfor %} + +# Update dependent list based on other packages requirements +if [[ -f /etc/sonic/${SERVICE}_dependent ]]; then + DEPENDENT="${DEPENDENT} $(cat /etc/sonic/${SERVICE}_dependent)" +fi + +if [[ -f /etc/sonic/${SERVICE}_multi_inst_dependent ]]; then + MULTI_INST_DEPENDENT="${MULTI_INST_DEPENDENT} cat /etc/sonic/${SERVICE}_multi_inst_dependent" +fi + +function debug() +{ + /usr/bin/logger $1 + /bin/echo `date` "- $1" >> ${DEBUGLOG} +} + +function check_warm_boot() +{ + SYSTEM_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|system" enable` + SERVICE_WARM_START=`$SONIC_DB_CLI STATE_DB hget "WARM_RESTART_ENABLE_TABLE|${SERVICE}" enable` + if [[ x"$SYSTEM_WARM_START" == x"true" ]] || [[ x"$SERVICE_WARM_START" == x"true" ]]; then + WARM_BOOT="true" +{#- TODO: restore count validation for SONiC packages #} + else + WARM_BOOT="false" + fi +} + +function check_fast_boot() +{ + if [[ $($SONIC_DB_CLI STATE_DB GET "FAST_REBOOT|system") == "1" ]]; then + FAST_BOOT="true" + else + FAST_BOOT="false" + fi +} + +function start_dependent_services() { + if [[ x"$WARM_BOOT" != x"true" ]]; then + for dep in ${DEPENDENT}; do + /bin/systemctl start ${dep} + done + for dep in ${MULTI_INST_DEPENDENT}; do + if [[ ! -z $DEV ]]; then + /bin/systemctl start ${dep}@$DEV + else + /bin/systemctl start ${dep} + fi + done + fi +} + +function stop_dependent_services() { + if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then + for dep in ${DEPENDENT}; do + /bin/systemctl stop ${dep} + done + for dep in ${MULTI_INST_DEPENDENT}; do + if [[ ! -z $DEV ]]; then + /bin/systemctl stop ${dep}@$DEV + else + /bin/systemctl stop ${dep} + fi + done + fi +} + +function start() { + debug "Starting ${SERVICE}$DEV service..." + + # start service docker + /usr/bin/${SERVICE}.sh start $DEV + debug "Started ${SERVICE}$DEV service..." + +{%- if manifest.service["post-start-action"] %} + docker exec -it ${SERVICE}${DEV} {{ manifest.service["post-start-action"] }} +{%- endif %} +} + +function wait() { + start_dependent_services + + if [[ ! -z $DEV ]]; then + /usr/bin/${SERVICE}.sh wait $DEV + else + /usr/bin/${SERVICE}.sh wait + fi +} + +function stop() { + debug "Stopping ${SERVICE}$DEV service..." + +{%- if manifest.service["pre-shutdown-action"] %} + docker exec -it ${SERVICE}${DEV} {{ manifest.service["pre-shutdown-action"] }} +{%- endif %} + + # For WARM/FAST boot do not perform service stop + if [[ x"$WARM_BOOT" != x"true" ]] && [[ x"$FAST_BOOT" != x"true" ]]; then + /usr/bin/${SERVICE}.sh stop $DEV + else + docker kill ${SERVICE}$DEV &> /dev/null || debug "Docker ${SERVICE}$DEV is not running ($?) ..." + fi + + debug "Stopped ${SERVICE}$DEV service..." + + stop_dependent_services +} + +OP=$1 +DEV=$2 + +check_warm_boot +check_fast_boot + +debug "Fast boot flag: ${SERVICE}$DEV ${FAST_BOOT}." +debug "Warm boot flag: ${SERVICE}$DEV ${WARM_BOOT}." + +case "$OP" in + start|wait|stop) + $1 + ;; + *) + echo "Usage: $0 {start|wait|stop}" + exit 1 + ;; +esac diff --git a/src/sonic-package-manager/sonic-package-manager-data/templates/sonic.service.j2 b/src/sonic-package-manager/sonic-package-manager-data/templates/sonic.service.j2 new file mode 100644 index 00000000000..72d6ab698c7 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/templates/sonic.service.j2 @@ -0,0 +1,39 @@ +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# +{%- set path = '/usr/local/bin' %} +{%- set multi_instance = multi_instance|default(False) %} +{%- set multi_instance_services = multi_instance_services|default([]) %} +[Unit] +Description={{ manifest.service.name }} container +{%- for service in manifest.service.requires %} +Requires={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.requisite %} +Requisite={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.after %} +After={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- for service in manifest.service.before %} +Before={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +BindsTo=sonic.target +After=sonic.target +StartLimitIntervalSec=1200 +StartLimitBurst=3 + +[Service] +ExecStartPre={{path}}/{{manifest.service.name}}.sh start{% if multi_instance %} %i{% endif %} +ExecStart={{path}}/{{manifest.service.name}}.sh wait{% if multi_instance %} %i{% endif %} +ExecStop={{path}}/{{manifest.service.name}}.sh stop{% if multi_instance %} %i{% endif %} +RestartSec=30 + +{%- if not manifest.service.delayed %} +[Install] +WantedBy=sonic.target +{%- for service in manifest.service["wanted-by"] %} +WantedBy={{ service }}{% if multi_instance and service in multi_instance_services %}@%i{% endif %}.service +{%- endfor %} +{%- endif %} diff --git a/src/sonic-package-manager/sonic-package-manager-data/templates/timer.unit.j2 b/src/sonic-package-manager/sonic-package-manager-data/templates/timer.unit.j2 new file mode 100644 index 00000000000..a757b8deb85 --- /dev/null +++ b/src/sonic-package-manager/sonic-package-manager-data/templates/timer.unit.j2 @@ -0,0 +1,15 @@ +# +# =============== Managed by SONiC Package Manager. DO NOT EDIT! =============== +# auto-generated from {{ source }} by sonic-package-manager +# +[Unit] +Description=Delays {{ manifest.service.name }} until SONiC has started +PartOf={{ manifest.service.name }}{% if multi_instance %}@%i{% endif %}.service + +[Timer] +OnUnitActiveSec=0 sec +OnBootSec=3min 30 sec +Unit={{ manifest.service.name }}{% if multi_instance %}@%i{% endif %}.service + +[Install] +WantedBy=timers.target sonic.target diff --git a/src/sonic-package-manager/sonic_package_manager/__init__.py b/src/sonic-package-manager/sonic_package_manager/__init__.py new file mode 100644 index 00000000000..9d8827c5e4d --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +from sonic_package_manager.manager import PackageManager + +__all__ = ['PackageManager'] diff --git a/src/sonic-package-manager/sonic_package_manager/constraint.py b/src/sonic-package-manager/sonic_package_manager/constraint.py new file mode 100644 index 00000000000..f7bc16779f7 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/constraint.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +""" Package version constraints module. """ +import re +from abc import ABC +from dataclasses import dataclass + +import semver + + +class VersionConstraint(semver.VersionConstraint, ABC): + """ Extends VersionConstraint from semver package. """ + + @staticmethod + def parse(constraint_expression: str) -> 'VersionConstraint': + """ Parse version constraint. + + Args: + constraint_expression: Expression syntax: "[[op][version]]+". + Returns: + The resulting VersionConstraint object. + """ + + return semver.parse_constraint(constraint_expression) + + +@dataclass +class PackageConstraint: + """ PackageConstraint is a package version constraint. """ + + name: str + constraint: VersionConstraint + + def __str__(self): + return f'{self.name}{self.constraint}' + + @staticmethod + def parse(constraint_expression: str) -> 'PackageConstraint': + """ Parse package constraint string which contains a package + name separated by a space with zero, one or more version constraint + expressions. A variety of version matching operators are supported + including >, <, ==, !=, ^, *. See Examples. + + Args: + constraint_expression: Expression syntax "[package name] [[op][version]]+". + + Returns: + PackageConstraint object. + + Examples: + >>> PackageConstraint.parse('syncd^1.0.0').constraint + =1.0.0,<2.0.0)> + >>> PackageConstraint.parse('swss>1.3.2 <4.2.1').constraint + 1.3.2,<4.2.1)> + >>> PackageConstraint.parse('swss').constraint + + """ + + REQUIREMENT_SPECIFIER_RE = \ + r'(?P[A-Za-z0-9_-]+)(?P.*)' + + match = re.match(REQUIREMENT_SPECIFIER_RE, constraint_expression) + if match is None: + raise ValueError(f'Invalid constraint {constraint_expression}') + groupdict = match.groupdict() + name = groupdict.get('name') + constraint = groupdict.get('constraint') or '*' + return PackageConstraint(name, VersionConstraint.parse(constraint)) + + diff --git a/src/sonic-package-manager/sonic_package_manager/database.py b/src/sonic-package-manager/sonic_package_manager/database.py new file mode 100644 index 00000000000..c25396b29c8 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/database.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python + +""" Repository Database interface module. """ +import json +from dataclasses import dataclass +from typing import Optional, Dict, Callable + +from sonic_package_manager.errors import PackageManagerError, PackageNotFoundError, PackageAlreadyExistsError +from sonic_package_manager.version import Version + +DB_FILE_PATH = '/var/lib/sonic-package-manager/packages.json' + + +@dataclass(order=True) +class PackageEntry: + """ Package database single entry object. """ + + name: str + repository: str + description: Optional[str] = None + default_reference: Optional[str] = None + version: Optional[Version] = None + installed: bool = False + built_in: bool = False + + +def package_from_dict(name: str, package_info: Dict) -> PackageEntry: + """ Parse dictionary into PackageEntry object.""" + + repository = package_info['repository'] + description = package_info.get('description') + default_reference = package_info.get('default-reference') + version = package_info.get('installed-version') + if version: + version = Version.parse(version) + installed = package_info.get('installed', False) + built_in = package_info.get('built-in', False) + + return PackageEntry(name, repository, description, + default_reference, version, installed, + built_in) + + +def package_to_dict(package: PackageEntry) -> Dict: + """ Serialize package into dictionary. """ + + return { + 'repository': package.repository, + 'description': package.description, + 'default-reference': package.default_reference, + 'installed-version': None if package.version is None else str(package.version), + 'installed': package.installed, + 'built-in': package.built_in, + } + + +class PackageDatabase: + """ An interface to SONiC repository database """ + + def __init__(self, + database: Dict[str, PackageEntry], + on_save: Optional[Callable] = None): + """ Initialize PackageDatabase. + + Args: + database: Database dictionary + on_save: Optional callback to execute on commit() + """ + + self._database = database + self._on_save = on_save + + def add_package(self, + name: str, + repository: str, + description: Optional[str] = None, + default_reference: Optional[str] = None): + """ Adds a new package entry in database. + + Args: + name: Package name. + repository: Repository URL. + description: Description string. + default_reference: Default version string. + + Raises: + PackageAlreadyExistsError: if package already exists in database. + """ + + if self.has_package(name): + raise PackageAlreadyExistsError(name) + + package = PackageEntry(name, repository, description, default_reference) + self._database[name] = package + + def remove_package(self, name: str): + """ Removes package entry from database. + + Args: + name: repository name. + Raises: + PackageNotFoundError: Raises when package with the given name does not exist + in the database. + """ + + pkg = self.get_package(name) + + if pkg.built_in: + raise PackageManagerError(f'Package {name} is built-in, cannot remove it') + + if pkg.installed: + raise PackageManagerError(f'Package {name} is installed, uninstall it first') + + self._database.pop(name) + + def update_package(self, pkg: PackageEntry): + """ Modify repository in the database. + + Args: + pkg: Repository object. + Raises: + PackageManagerError: Raises when repository with the given name does not exist + in the database. + """ + + name = pkg.name + + if not self.has_package(name): + raise PackageNotFoundError(name) + + self._database[name] = pkg + + def get_package(self, name: str) -> PackageEntry: + """ Return a package referenced by name. + If the package is not found PackageNotFoundError is thrown. + + Args: + name: Package name. + Returns: + PackageInfo object. + Raises: + PackageNotFoundError: When package called name was not found. + """ + + try: + pkg = self._database[name] + except KeyError: + raise PackageNotFoundError(name) + + return pkg + + def has_package(self, name: str) -> bool: + """ Checks if the database contains an entry for a package. + called name. Returns True if the package exists, otherwise False. + + Args: + name: Package name. + Returns: + True if the package exists, otherwise False. + """ + + try: + self.get_package(name) + return True + except PackageNotFoundError: + return False + + def __iter__(self): + """ Iterates over packages in the database. + + Yields: + PackageInfo object. + """ + + for name, _ in self._database.items(): + yield self.get_package(name) + + @staticmethod + def from_file(db_file=DB_FILE_PATH) -> 'PackageDatabase': + """ Read database content from file. """ + + def on_save(database): + with open(db_file, 'w') as db: + db_content = {} + for name, package in database.items(): + db_content[name] = package_to_dict(package) + json.dump(db_content, db, indent=4) + + database = {} + with open(db_file) as db: + db_content = json.load(db) + for key in db_content: + package = package_from_dict(key, db_content[key]) + database[key] = package + return PackageDatabase(database, on_save) + + def commit(self): + """ Save database content to file. """ + + if self._on_save: + self._on_save(self._database) diff --git a/src/sonic-package-manager/sonic_package_manager/dockerapi.py b/src/sonic-package-manager/sonic_package_manager/dockerapi.py new file mode 100644 index 00000000000..9f531023013 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/dockerapi.py @@ -0,0 +1,179 @@ +#!/usr/bin/evn python + +""" Module provides Docker interface. """ + +import contextlib +import io +import os +import tarfile +from typing import Optional + +from sonic_package_manager.logger import log +from sonic_package_manager.progress import ProgressManager + + +def is_digest(ref: str): + return ref.startswith('sha256:') + + +def bytes_to_mb(bytes): + return bytes / 1024 / 1024 + + +def get_id(line): + return line['id'] + + +def get_status(line): + return line['status'] + + +def get_progress(line): + progress = line['progressDetail'] + current = bytes_to_mb(progress['current']) + total = bytes_to_mb(progress['total']) + return current, total + + +class DockerApi: + """ DockerApi provides a set of methods - + wrappers around docker client methods """ + + def __init__(self, client): + self.client = client + + def pull(self, repository: str, + reference: str, + tag: Optional[str] = None, + progress_manager: Optional[ProgressManager] = None): + """ Docker 'pull' command. + Args: + repository: repository to pull + reference: tag or digest + tag: tag the image right after pull with this tag + progress_manager: ProgressManager instance + """ + + if is_digest(reference): + image = f'{repository}@{reference}' + else: + image = f'{repository}:{reference}' + + log.debug(f'pulling image {image}') + + api = self.client.api + + with progress_manager or contextlib.nullcontext(): + for line in api.pull(repository, + reference, + stream=True, + decode=True): + log.debug(f'pull status: {line}') + if progress_manager is None: + continue + try: + id = get_id(line) + status = get_status(line) + if id not in progress_manager: + _, total = get_progress(line) + progress_manager.new(id, + total=total, + unit='Mb', + desc=f'{status} {id}') + pbar = progress_manager.get(id) + + # Complete status + if 'complete' in status: + pbar.desc = f'{status} {id}' + pbar.update(pbar.total) + continue + + current, total = get_progress(line) + # Status changed + if status not in pbar.desc: + pbar.desc = f'{status} {id}' + pbar.total = total + pbar.count = 0 + + pbar.update(current - pbar.count) + except KeyError: + # not a progress line + pass + + log.debug(f'image {image} pulled successfully') + + # Reference might be provided as a digest. In this case + # Docker will create a dangling image without any tag. + # This is not desired and may leave garbage Dockers. + # We are tagging it with a tag provided by the user. + if tag: + log.debug(f'tagging {image} with {repository}:{tag}') + api.tag(f'{image}', repository, tag) + + def rmi(self, repository: str, tag: str): + """ Docker 'rmi -f' command. """ + + image = f'{repository}:{tag}' + log.debug(f'removing image {image}') + + self.client.images.remove(f'{image}', force=True) + + log.debug(f'image {image} removed successfully') + + def rm(self, repository: str, tag: str): + """ Docker 'rm' command but removes by image and tag. """ + + for container in self.client.containers.list(all=True): + container_image = container.attrs['Config']['Image'] + if container_image == f'{repository}:{tag}': + container.remove(force=True) + log.debug(f'removed container {container.name}') + + def tag(self, src_img: str, + src_tag: str, + target_img: str, + target_tag: str): + """ Docker 'tag' command """ + + src = f'{src_img}:{src_tag}' + dst = f'{target_img}:{target_tag}' + + log.debug(f'tagging image {src} to {dst}') + + image = self.client.images.get(src) + image.tag(target_img, target_tag, force=True) + + log.debug(f'image {src} to {dst} tagged successfully') + + def labels(self, repository: str, tag: str): + """ Returns a list of labels associated with image. """ + + image = f'{repository}:{tag}' + log.debug(f'inspecting image labels {image}') + + labels = self.client.images.get(image).labels + + log.debug(f'image {image} labels successfully: {labels}') + return labels + + def cp(self, repository: str, tag: str, src_path: str, dst_path: str): + """ Copy src_path from the docker image to host dst_path. """ + + image = f'{repository}:{tag}' + buf = bytes() + + container = self.client.containers.create(image) + try: + bits, _ = container.get_archive(src_path) + for chunk in bits: + buf += chunk + finally: + container.remove(force=True) + + with tarfile.open(fileobj=io.BytesIO(buf)) as tar: + for member in tar: + if dst_path.endswith('/'): + tar.extract(member, dst_path) + else: + member.name = dst_path + tar.extract(member, dst_path) diff --git a/src/sonic-package-manager/sonic_package_manager/errors.py b/src/sonic-package-manager/sonic_package_manager/errors.py new file mode 100644 index 00000000000..359b67ddb97 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/errors.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +""" SONiC Package Manager exceptions are defined in this module. """ + +from dataclasses import dataclass +from typing import Optional + +from sonic_package_manager.constraint import PackageConstraint +from sonic_package_manager.version import Version + + +class PackageManagerError(Exception): + """ Base class for exceptions generated by SONiC package manager """ + + pass + + +class ManifestError(Exception): + """ Class for manifest validate failures. """ + + pass + + +@dataclass +class PackageNotFoundError(PackageManagerError): + """ Repository not found in repository database exception """ + + name: str + + def __str__(self): + return f'Package {self.name} is not found in packages database' + + +@dataclass +class PackageAlreadyExistsError(PackageManagerError): + """ Package already exists in the packages database exception. """ + + name: str + + def __str__(self): + return f'Package {self.name} already exists in packages database' + + +class PackageInstallationError(PackageManagerError): + """ Exception for package installation error. """ + + pass + + +class PackageUninstallationError(PackageManagerError): + """ Exception for package installation error. """ + + pass + + +class PackageUpgradeError(PackageManagerError): + """ Exception for package upgrade error. """ + + pass + + +@dataclass +class PackageSonicRequirementError(PackageInstallationError): + """ Exception for installation errors, when SONiC version requirement is not met. """ + + name: str + constraint: PackageConstraint + installed_ver: Version + + def __str__(self): + return (f'Package {self.name} requires base OS compatibility version {self.constraint} ' + f'while the installed version is {self.installed_ver}') + + +@dataclass +class PackageDependencyError(PackageInstallationError): + """ Exception class for installation errors related to missing dependency. """ + + name: str + constraint: PackageConstraint + installed_ver: Optional[Version] = None + + def __str__(self): + if self.installed_ver: + return (f'Package {self.name} requires {self.constraint} ' + f'but version {self.installed_ver} is installed') + return f'Package {self.name} requires {self.constraint} but it is not installed' + + +@dataclass +class PackageConflictError(PackageInstallationError): + """ Exception class for installation errors related to missing dependency. """ + + name: str + constraint: PackageConstraint + installed_ver: Version + + def __str__(self): + return (f'Package {self.name} conflicts with {self.constraint} but ' + f'version {self.installed_ver} is installed') diff --git a/src/sonic-package-manager/sonic_package_manager/logger.py b/src/sonic-package-manager/sonic_package_manager/logger.py new file mode 100644 index 00000000000..3d5e06d35f2 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/logger.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +""" Logger for sonic-package-manager. """ + +import logging.handlers + +import click_log + + +class Formatter(click_log.ColorFormatter): + """ Click logging formatter. """ + + colors = { + 'error': dict(fg='red'), + 'exception': dict(fg='red'), + 'critical': dict(fg='red'), + 'debug': dict(fg='blue', bold=True), + 'warning': dict(fg='yellow'), + } + + +log = logging.getLogger("sonic-package-manager") +log.setLevel(logging.INFO) + +click_handler = click_log.ClickHandler() +click_handler.formatter = Formatter() + +log.addHandler(click_handler) +log.addHandler(logging.handlers.SysLogHandler()) diff --git a/src/sonic-package-manager/sonic_package_manager/main.py b/src/sonic-package-manager/sonic_package_manager/main.py new file mode 100644 index 00000000000..081b9c9a703 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/main.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python + +import functools +import json +import os +import sys +import typing + +import click +import click_log +import tabulate +from natsort import natsorted + +from sonic_package_manager.database import PackageEntry, PackageDatabase +from sonic_package_manager.errors import PackageManagerError +from sonic_package_manager.logger import log +from sonic_package_manager.manager import PackageManager, parse_reference_expression + +BULLET_UC = '\u2022' + + +def exit_cli(*args, **kwargs): + """ Print a message and exit with rc 1. """ + + click.secho(*args, **kwargs) + sys.exit(1) + + +def show_help(ctx): + click.echo(ctx.get_help()) + ctx.exit(0) + + +def root_privileges_required(func: typing.Callable) -> typing.Callable: + """ Decorates a function, so that the function is invoked + only if the user is root. """ + + @functools.wraps(func) + def wrapped_function(*args, **kwargs): + """ Wrapper around func. """ + + if os.geteuid() != 0: + exit_cli('Root privileges required for this operation', fg='red') + + return func(*args, **kwargs) + + return wrapped_function + + +def get_package_status(package: PackageEntry): + if package.built_in: + return 'Built-In' + elif package.installed: + return 'Installed' + else: + return 'Not Installed' + + +@click.group() +@click.pass_context +def cli(ctx): + """ SONiC Package Manager. """ + + ctx.obj = PackageManager.get_manager() + + +@cli.group() +@click.pass_context +def repository(ctx): + """ Repository management commands. """ + + pass + + +@cli.group() +@click.pass_context +def package(ctx): + """ SONiC Package commands. """ + + pass + + +@package.group() +@click.pass_context +def show(ctx): + """ Package show CLI commands. """ + + pass + + +@cli.command() +@click.pass_context +def list(ctx): + """ List available repositories. """ + + table_header = ["Name", "Repository", "Description", "Version", "Status"] + table_body = [] + + manager: PackageManager = ctx.obj + + try: + db = manager.get_database() + for package in natsorted(db): + version = package.version or 'N/A' + status = get_package_status(package) + + table_body.append([ + package.name, + package.repository, + package.description, + version, + status + ]) + + click.echo(tabulate.tabulate(table_body, table_header)) + except PackageManagerError as err: + exit_cli(f'Failed to list repositories: {err}', fg='red') + + +@show.command() +@click.argument('expression') +@click.pass_context +def manifest(ctx, expression): + """ Print the manifest content. """ + + manager: PackageManager = ctx.obj + + try: + package_reference = parse_reference_expression(expression) + package = manager.get_package(package_reference.name, + package_reference.reference) + click.echo(json.dumps(package.manifest.unmarshal(), indent=4)) + except Exception as err: + exit_cli(f'Failed to describe {expression}: {err}', fg='red') + + +@show.command() +@click.argument('name') +@click.option('--all', is_flag=True, help='Show all available tags in repository') +@click.option('--plain', is_flag=True, help='Plain output') +@click.pass_context +def versions(ctx, name, all, plain): + """ Print available versions. """ + + try: + manager: PackageManager = ctx.obj + versions = manager.get_available_versions(name, all) + for version in versions: + if not plain: + click.secho(f'{BULLET_UC} ', bold=True, fg='green', nl=False) + click.secho(f'{version}') + except Exception as err: + exit_cli(f'Failed to get package versions {name}: {err}', fg='red') + + +@show.command() +@click.argument('expression') +@click.pass_context +def changelog(ctx, expression): + """ Print the package changelog. """ + + manager: PackageManager = ctx.obj + + try: + ref = parse_reference_expression(expression) + package = manager.get_package(ref.name, ref.reference) + changelog = package.manifest['package']['changelog'] + + if not changelog: + raise PackageManagerError(f'No changelog for package {expression}') + + for version, entry in changelog.items(): + author = entry.get('author') or 'N/A' + email = entry.get('email') or 'N/A' + changes = entry.get('changes') or [] + date = entry.get('date') or 'N/A' + click.secho(f'{version}:\n', fg='green', bold=True) + for line in changes: + click.secho(f' {BULLET_UC} {line}', bold=True) + click.secho(f'\n {author} ' + f'({email}) {date}', fg='green', bold=True) + click.secho('') + + except Exception as err: + exit_cli(f'Failed to print package changelog: {err}', fg='red') + + +@repository.command() +@click.argument('name', type=str) +@click.argument('repository', type=str) +@click.option('--default-reference', type=str) +@click.option('--description', type=str) +@click.pass_context +@root_privileges_required +def add(ctx, name, repository, default_reference, description): + """ Add a new repository to database. """ + + manager: PackageManager = ctx.obj + + try: + manager.add_package(name, repository, description=description, default_reference=default_reference) + except Exception as err: + exit_cli(f'Failed to add repository {name}: {err}', fg='red') + + +@repository.command() +@click.argument("name") +@click.pass_context +@root_privileges_required +def remove(ctx, name): + """ Remove a package from database. """ + + manager: PackageManager = ctx.obj + + try: + manager.remove_package(name) + except Exception as err: + exit_cli(f'Failed to remove repository {name}: {err}', fg='red') + + +@cli.command() +@click.option('-f', '--force', is_flag=True) +@click.option('-y', '--yes', is_flag=True) +@click.argument('expression') +@click.pass_context +@click_log.simple_verbosity_option(log) +@root_privileges_required +def install(ctx, expression, force, yes): + """ Install a package. """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm(f'Package {expression} is going to be installed, ' + f'continue?', abort=True, show_default=True) + + try: + manager.install(expression, force) + except Exception as err: + exit_cli(f'Failed to install package {expression}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@click.option('-f', '--force', is_flag=True) +@click.option('-y', '--yes', is_flag=True) +@click.argument('expression') +@click.pass_context +@click_log.simple_verbosity_option(log) +@root_privileges_required +def upgrade(ctx, expression, force, yes): + """ Upgrade a package. """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm(f'Package is going to be upgraded to {expression}, ' + f'continue?', abort=True, show_default=True) + + try: + manager.upgrade(expression, force) + except Exception as err: + exit_cli(f'Failed to upgrade package {expression}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@click.option('-f', '--force', is_flag=True) +@click.option('-y', '--yes', is_flag=True) +@click.argument('name') +@click.pass_context +@click_log.simple_verbosity_option(log) +@root_privileges_required +def uninstall(ctx, name, force, yes): + """ Uninstall a package. """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm(f'Package {name} is going to be uninstalled, ' + f'continue?', abort=True, show_default=True) + + try: + manager.uninstall(name, force) + except Exception as err: + exit_cli(f'Failed to uninstall package {name}: {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +@cli.command() +@click.option('-f', '--force', is_flag=True) +@click.option('-y', '--yes', is_flag=True) +@click.argument('database', type=click.Path()) +@click.pass_context +@click_log.simple_verbosity_option(log) +@root_privileges_required +def migrate(ctx, database, force, yes): + """ Migrate SONiC packages from the given database file. """ + + manager: PackageManager = ctx.obj + + if not yes and not force: + click.confirm('Continue with package migration?', abort=True, show_default=True) + + try: + manager.migrate_packages(PackageDatabase.from_file(database)) + except Exception as err: + exit_cli(f'Failed to migrate packages {err}', fg='red') + except KeyboardInterrupt: + exit_cli(f'Operation canceled by user', fg='red') + + +if __name__ == "__main__": + cli() diff --git a/src/sonic-package-manager/sonic_package_manager/manager.py b/src/sonic-package-manager/sonic_package_manager/manager.py new file mode 100644 index 00000000000..8f9d3cfaad5 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/manager.py @@ -0,0 +1,582 @@ +#!/usr/bin/env python +import contextlib +import functools +import os +from typing import Optional, Union, Dict, Any + +import docker +from sonic_py_common import device_info + +from sonic_package_manager import utils +from sonic_package_manager.constraint import PackageConstraint +from sonic_package_manager.database import PackageDatabase, PackageEntry +from sonic_package_manager.dockerapi import DockerApi +from sonic_package_manager.errors import ( + PackageInstallationError, + PackageDependencyError, + PackageConflictError, + PackageSonicRequirementError, + PackageManagerError, + PackageUninstallationError, PackageUpgradeError +) +from sonic_package_manager.logger import log +from sonic_package_manager.manifest_resolver import ManifestResolver +from sonic_package_manager.package import Package +from sonic_package_manager.progress import ProgressManager +from sonic_package_manager.reference import PackageReference +from sonic_package_manager.registry import RegistryResolver +from sonic_package_manager.service_creator.creator import ServiceCreator, run_command +from sonic_package_manager.service_creator.feature import FeatureRegistry +from sonic_package_manager.service_creator.sonic_db import SonicDB +from sonic_package_manager.version import Version, VersionRange, version_to_tag, tag_to_version + + +@contextlib.contextmanager +def failure_ignore(ignore: bool): + """ Ignores failures based on parameter passed. """ + try: + yield + except Exception as err: + if ignore: + log.warning(f'ignoring error {err}') + else: + raise + + +def parse_package_reference_from_constraint(constraint: PackageConstraint) -> PackageReference: + package_name, version_constraint = constraint.name, constraint.constraint + # Allow only specific version for now. + # Later we can improve package manager to support + # installing packages using expressions like 'package>1.0.0' + if version_constraint == VersionRange(): # empty range means any version + return PackageReference(package_name, None) + if not isinstance(version_constraint, Version): + raise PackageManagerError(f'Can only install specific version. ' + f'Use only following expression "{package_name}==" ' + f'to install specific version') + return PackageReference(package_name, version_to_tag(version_constraint)) + + +def parse_reference_expression(expression): + try: + return parse_package_reference_from_constraint(PackageConstraint.parse(expression)) + except ValueError: + # if we failed to parse the expression as constraint expression + # we will try to parse it as reference + return PackageReference.parse(expression) + + +def check_dependencies_and_conflicts(packages: Dict[str, Package]): + """ Verify that all dependencies are met in all packages passed to this function. + Args: + packages: list of packages to check + Raises: + PackageDependencyError: if dependency is missing + PackageConflictError: if there is a conflict between packages + """ + + for name, package in packages.items(): + log.debug(f'checking dependencies for {name}') + for dependency in package.manifest['package']['depends']: + dependency_package = packages.get(dependency.name) + if dependency_package is None: + raise PackageDependencyError(package.name, dependency) + installed_version = dependency_package.version + log.debug(f'dependency package is installed {dependency.name}: {installed_version}') + if not dependency.constraint.allows_all(installed_version): + raise PackageDependencyError(package.name, dependency, installed_version) + + log.debug(f'checking conflicts for {name}') + for conflict in package.manifest['package']['breaks']: + conflicting_package = packages.get(conflict.name) + if conflicting_package is None: + continue + installed_version = conflicting_package.version + log.debug(f'conflicting package is installed {conflict.name}: {installed_version}') + if conflict.constraint.allows_all(installed_version): + raise PackageConflictError(package.name, conflict, installed_version) + + +class PackageManager: + """ SONiC Package Manager. """ + + HOST_PLUGIN_PATH = '/usr/local/lib/python2.7/dist-packages/{}/plugins/{}' + + def __init__(self, + docker: DockerApi, + registry_resolver: RegistryResolver, + database: PackageDatabase, + manifest_resolver: ManifestResolver, + service_creator: ServiceCreator, + device_information: Any): + """ Initialize PackageManager. """ + + self.docker = docker + self.registry_resolver = registry_resolver + self.database = database + self.manifest_resolver = manifest_resolver + self.service_creator = service_creator + self.feature_registry = service_creator.feature_registry + self.is_multi_npu = device_information.is_multi_npu() + self.num_npus = device_information.get_num_npus() + self.version_info = device_information.get_sonic_version_info() + self.base_os_version = Version.parse(self.version_info.get('base-os-compatibility-version')) + + def install(self, expression: str, force=False): + """ Install a SONiC Package from the package reference + expression string. Can force the installation if force parameter + is True. + + Args: + expression: SONiC Package reference expression + force: Force the installation. + Raises: + PackageManagerError + """ + + package_reference = parse_reference_expression(expression) + name, reference = package_reference.name, package_reference.reference + + with failure_ignore(force): + if self.is_installed(name): + raise PackageInstallationError(f'{name} is already installed') + + package = self.get_package(name, reference) + version_constraint = package.manifest['package']['base-os-constraint'] + + with failure_ignore(force): + if not version_constraint.allows_all(self.base_os_version): + raise PackageSonicRequirementError(package.name, version_constraint, + self.base_os_version) + + installed_packages = self.get_installed_packages_and(package) + + with failure_ignore(force): + check_dependencies_and_conflicts(installed_packages) + + # After all checks are passed we proceed to actual installation + + version = package.manifest['package']['version'] + cleanup = [] # cleanup function list + + def add_cleanup(func): + cleanup.append(func) + + def exec_cleanup(): + for cleanup_func in reversed(cleanup): + try: + cleanup_func() + except Exception as cleanup_err: + log.error(f'{cleanup_err}') + + try: + self.docker.pull(package.repository, package.reference, version, ProgressManager()) + add_cleanup(functools.partial(self.docker.rmi, package.repository, version)) + + self.docker.tag(package.repository, version, package.repository, 'latest') + add_cleanup(functools.partial(self.docker.rmi, package.repository, 'latest')) + + self.service_creator.create(package) + add_cleanup(functools.partial(self.service_creator.remove, package)) + + self.install_cli_plugins(package) + add_cleanup(functools.partial(self.uninstall_cli_plugins, package)) + except Exception as err: + exec_cleanup() + raise PackageInstallationError(f'Failed to install {package.name}: {err}') + except KeyboardInterrupt: + exec_cleanup() + raise + + package_entry = package.entry + package_entry.installed = True + self.database.update_package(package_entry) + self.database.commit() + + def uninstall(self, name: str, force=False): + """ Uninstall a SONiC Package referenced by name. + The uninstallation can be forced if force argument is + True. + + Args: + name: SONiC Package name. + force: Force the installation. + Raises: + PackageManagerError + """ + + with failure_ignore(force): + if not self.is_installed(name): + raise PackageUninstallationError(f'{name} is not installed') + + with failure_ignore(force): + if self.feature_registry.is_feature_enabled(name): + raise PackageUninstallationError(f'{name} is enabled. Disable the feature first') + + package = self.get_package(name) + + if package.built_in: + raise PackageUninstallationError(f'Cannot uninstall built-in package {package.name}') + + installed_packages = self.get_installed_packages_except(package) + + with failure_ignore(force): + check_dependencies_and_conflicts(installed_packages) + + # After all checks are passed we proceed to actual uninstallation + + try: + self.uninstall_cli_plugins(package) + self.service_creator.remove(package) + self.docker.rm(package.repository, package.reference) + self.docker.rm(package.repository, 'latest') + self.docker.rmi(package.repository, package.reference) + self.docker.rmi(package.repository, 'latest') + except Exception as err: + raise PackageUninstallationError(f'Failed to uninstall {package.name}: {err}') + + package_entry = package.entry + package_entry.installed = False + package_entry.version = None + self.database.update_package(package_entry) + self.database.commit() + + def upgrade(self, expression: str, force=False): + """ Upgrade a SONiC Package to a version the package reference + expression specifies. Can force the upgrade if force parameter + is True. Force can allow a package downgrade. + + Args: + expression: SONiC Package reference expression + force: Force the installation. + Raises: + PackageManagerError + """ + + package_reference = parse_reference_expression(expression) + name, reference = package_reference.name, package_reference.reference + + with failure_ignore(force): + if not self.is_installed(name): + raise PackageUpgradeError(f'{name} is not installed') + + old_package = self.get_package(name) + new_package = self.get_package(name, reference) + + if old_package.built_in: + raise PackageUpgradeError(f'Cannot upgrade built-in package {old_package.name}') + + old_feature = old_package.manifest['service']['name'] + new_feature = new_package.manifest['service']['name'] + old_version = old_package.manifest['package']['version'] + new_version = new_package.manifest['package']['version'] + + with failure_ignore(force): + if old_version == new_version: + raise PackageUpgradeError(f'{new_version} is already installed') + + # TODO: Not all packages might support downgrade. + # We put a check here but we understand that for some packages + # the downgrade might be safe to do. In that case we might want to + # add another argument to this function: allow_downgrade: bool = False. + # Another way to do that might be a variable in manifest describing package + # downgrade ability or downgrade-able versions. + if new_version < old_version: + raise PackageUpgradeError(f'Request to downgrade from {old_version} to {new_version}. ' + f'Downgrade might be not supported by the package') + + version_constraint = new_package.manifest['package']['base-os-constraint'] + + with failure_ignore(force): + if not version_constraint.allows_all(self.base_os_version): + raise PackageSonicRequirementError(new_package.name, version_constraint, + self.base_os_version) + + # remove currently installed package from the list + installed_packages = self.get_installed_packages_except(old_package) + # add the package to upgrade to the list + installed_packages[new_package.name] = new_package + + with failure_ignore(force): + check_dependencies_and_conflicts(installed_packages) + + # After all checks are passed we proceed to actual upgrade + + cleanup = [] # cleanup function list + + def add_cleanup(func): + cleanup.append(func) + + def exec_cleanup(): + for cleanup_func in reversed(cleanup): + try: + cleanup_func() + except Exception as cleanup_err: + log.error(f'{cleanup_err}') + + try: + self.uninstall_cli_plugins(old_package) + add_cleanup(functools.partial(self.install_cli_plugins, old_package)) + self.docker.pull(new_package.repository, new_package.reference, new_version, ProgressManager()) + add_cleanup(functools.partial(self.docker.rmi, new_package.repository, new_version)) + + if self.feature_registry.is_feature_enabled(old_feature): + self.systemctl_action(old_package, 'stop') + add_cleanup(functools.partial(self.systemctl_action, old_package, 'start')) + + self.service_creator.remove(old_package, deregister_feature=False) + add_cleanup(functools.partial(self.service_creator.create, old_package, register_feature=False)) + + # This is no return point, after we start removing old Docker images + # there is no guaranty we can actually successfully roll-back. + + self.docker.rm(old_package.repository, old_package.reference) + self.docker.rm(old_package.repository, 'latest') + self.docker.rmi(old_package.repository, old_package.reference) + self.docker.rmi(old_package.repository, 'latest') + + self.docker.tag(new_package.repository, new_version, new_package.repository, 'latest') + add_cleanup(functools.partial(self.docker.rmi, old_package.repository, 'latest')) + + self.service_creator.create(new_package, register_feature=False) + add_cleanup(functools.partial(self.service_creator.remove, new_package, deregister_feature=False)) + + if self.feature_registry.is_feature_enabled(new_feature): + self.systemctl_action(new_package, 'start') + + self.install_cli_plugins(new_package) + except Exception as err: + exec_cleanup() + raise PackageUpgradeError(f'Failed to upgrade {new_package.name}: {err}') + except KeyboardInterrupt: + exec_cleanup() + raise + + new_package_entry = new_package.entry + new_package_entry.installed = True + self.database.update_package(new_package_entry) + self.database.commit() + + def migrate_packages(self, old_package_database: PackageDatabase): + """ Migrate packages from old database. This function can + do a comparison between current database and the database + passed in as argument. + If the package is missing in the current database it will be added. + If the package is installed in the passed database and in the current + it is not installed it will be installed with a passed database package version. + If the package is installed in the passed database and it is installed + in the current database but with older version the package will be upgraded to + the never version. + If the package is installed in the passed database and in the current + it is installed but with never version - no actions are taken. + + Args: + old_package_database: SONiC Package Database to migrate packages from. + Raises: + PackageManagerError + """ + + self.migrate_package_database(old_package_database) + + # TODO: Topological sort packages by their dependencies first. + for old_package in old_package_database: + if not old_package.installed or old_package.built_in: + continue + + log.info(f'migrating package {old_package.name}') + + new_package = self.database.get_package(old_package.name) + if new_package.installed or new_package.default_reference is not None: + pkg = self.get_package(new_package.name) + new_package_version = pkg.manifest['package']['version'] + + if old_package.version > new_package_version: + log.info(f'old package version is greater then default version in new image: ' + f'{old_package.version} > {new_package_version}') + if new_package.installed: + log.info(f'upgrading {new_package.name} to {old_package.version}') + self.upgrade(f'{new_package.name}=={old_package.version}') + else: + log.info(f'installing {new_package.name} version {old_package.version}') + self.install(f'{new_package.name}=={old_package.version}') + else: + if not new_package.installed: + log.info(f'installing {new_package.name} version {new_package_version}') + self.install(f'{new_package.name}=={new_package_version}') + else: + log.info(f'skipping {new_package.name} as installed version is newer') + else: + # No default version and package is not installed. + log.info(f'installing {new_package.name} version {old_package.version}') + self.install(f'{new_package.name}=={old_package.version}') + + self.database.commit() + + def migrate_package_database(self, old_package_database: PackageDatabase): + for package in old_package_database: + if not self.has_package(package.name): + self.database.add_package(package.name, + package.repository, + package.description, + package.default_reference) + + def get_database(self): + return self.database + + def get_docker(self): + return self.docker + + def add_package(self, *args, **kwargs): + self.database.add_package(*args, **kwargs) + self.database.commit() + + def remove_package(self, name: str): + self.database.remove_package(name) + self.database.commit() + + def get_package(self, name: str, ref: Optional[Union[Version, str]] = None) -> Package: + """ Get package from name and reference. If reference is not provided + reference returned by get_package_default_reference is used. + + Args: + name: Package name + ref: Optional reference to use + Returns: + Package object + """ + + package_entry = self.get_database().get_package(name) + if ref is None: + ref = self.get_package_default_reference(package_entry) + else: + if str(package_entry.version) != ref: + package_entry.installed = False + + manifest = self.get_manifest(package_entry, ref) + package_entry.version = manifest['package']['version'] + return Package(package_entry, ref, manifest) + + @staticmethod + def get_package_default_reference(package_entry: PackageEntry) -> str: + """ Returns default reference for the package. + If package is installed the installed tag, a version, is returned. + If package is not installed the default reference from package + database is used. + + Args: + package_entry: Package Database Entry + Returns: + Reference string + """ + + if package_entry.installed: + return str(package_entry.version) + + if package_entry.default_reference is not None: + return package_entry.default_reference + + raise PackageManagerError(f'No default reference tag. ' + f'Please specify the version or digest explicitly') + + def get_available_versions(self, name: str, all: bool = False): + package_info = self.database.get_package(name) + registry = self.registry_resolver.get_registry_for(package_info.repository) + available_tags = registry.tags(package_info.repository) + + def is_semantic_ver_tag(tag: str) -> bool: + try: + tag_to_version(tag) + return True + except ValueError: + pass + return False + + if all: + return available_tags + + return map(tag_to_version, filter(is_semantic_ver_tag, available_tags)) + + def get_installed_packages_and(self, package: Package) -> Dict[str, Package]: + packages = self.get_installed_packages() + packages[package.name] = package + return packages + + def get_installed_packages_except(self, package: Package) -> Dict[str, Package]: + packages = self.get_installed_packages() + packages.pop(package.name) + return packages + + def is_installed(self, name: str) -> bool: + if not self.database.has_package(name): + return False + package_info = self.database.get_package(name) + return package_info.installed + + def has_package(self, name: str): + return self.database.has_package(name) + + def get_installed_packages(self): + return {entry.name: self.get_package(entry.name, str(entry.version)) + for entry in self.get_database() if entry.installed} + + def get_manifest(self, package_info: PackageEntry, ref: str): + return self.manifest_resolver.get_manifest(package_info, ref) + + def systemctl_action(self, package: Package, action: str): + name = package.manifest['service']['name'] + host_service = package.manifest['service']['host-service'] + asic_service = package.manifest['service']['asic-service'] + single_instance = host_service or (asic_service and not self.is_multi_npu) + multi_instance = asic_service and self.is_multi_npu + + if single_instance: + run_command(f'systemctl {action} {name}') + if multi_instance: + for npu in range(self.num_npus): + run_command(f'systemctl {action} {name}@{npu}') + + @staticmethod + def get_cli_plugin_name(package: Package): + return utils.make_python_identifier(package.name) + '.py' + + @classmethod + def get_cli_plugin_path(cls, package: Package, command): + return cls.HOST_PLUGIN_PATH.format( + command, + cls.get_cli_plugin_name(package) + ) + + def install_cli_plugins(self, package: Package): + for command in ('show', 'config', 'clear'): + self.install_cli_plugin(package, command) + + def uninstall_cli_plugins(self, package: Package): + for command in ('show', 'config', 'clear'): + self.uninstall_cli_plugin(package, command) + + def install_cli_plugin(self, package: Package, command: str): + image_plugin_path = package.manifest['cli'][command] + host_plugin_path = self.get_cli_plugin_path(package, command) + repo = package.repository + tag = str(package.version) + if image_plugin_path: + self.docker.cp(repo, tag, image_plugin_path, host_plugin_path) + + def uninstall_cli_plugin(self, package: Package, command: str): + image_plugin_path = package.manifest['cli'][command] + if not image_plugin_path: + return + host_plugin_path = self.get_cli_plugin_path(package, command) + if os.path.exists(host_plugin_path): + os.remove(host_plugin_path) + + @staticmethod + def get_manager() -> 'PackageManager': + docker_api = DockerApi(docker.from_env()) + registry_resolver = RegistryResolver() + return PackageManager(DockerApi(docker.from_env()), + registry_resolver, + PackageDatabase.from_file(), + ManifestResolver(docker_api, registry_resolver), + ServiceCreator(FeatureRegistry(SonicDB)), + device_info) diff --git a/src/sonic-package-manager/sonic_package_manager/manifest.py b/src/sonic-package-manager/sonic_package_manager/manifest.py new file mode 100644 index 00000000000..54d9cec9337 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/manifest.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +from dataclasses import dataclass +from typing import Optional, List, Dict, Any, Callable + +from sonic_package_manager.constraint import PackageConstraint, VersionConstraint +from sonic_package_manager.errors import ManifestError +from sonic_package_manager.version import Version, VersionRange + + +class ManifestSchema: + """ ManifestSchema class describes and provides marshalling + and unmarshalling methods. + """ + + @dataclass + class ManifestNode: + """ + Base class for any manifest object. + + Attrs: + key: String representing the key for this object. + """ + + key: str + + def marshal(self, value): + """ Validates and returns a valid manifest dictionary. + + Args: + value: input value to validate. + Returns: valid manifest node. + """ + + raise NotImplementedError + + def unmarshal(self, value): + """ Un-marshals the manifest to a dictionary. + + Args: + value: input value to validate. + Returns: valid manifest node. + """ + + raise NotImplementedError + + @dataclass + class ManifestRoot(ManifestNode): + items: List + + def marshal(self, value: Optional[dict]): + result = {} + if value is None: + value = {} + + for item in self.items: + next_value = value.get(item.key) + result[item.key] = item.marshal(next_value) + return result + + def unmarshal(self, value): + return_value = {} + for item in self.items: + return_value[item.key] = item.unmarshal(value[item.key]) + return return_value + + @dataclass + class ManifestField(ManifestNode): + type: Callable + default: Optional[Any] = None + + def marshal(self, value): + if value is None: + if self.default is not None: + return self.default + raise ManifestError(f'{self.key} is a required field but it is missing') + try: + return_value = self.type(value) + except Exception as err: + raise ManifestError(f'Failed to convert {self.key}={value} to type {self.type.__name__}: {err}') + return return_value + + def unmarshal(self, value): + return str(value) + + @dataclass + class ManifestArray(ManifestNode): + type: Any + + def marshal(self, value): + if value is None: + return [] + + def marshal(v): + if isinstance(self.type, ManifestSchema.ManifestNode): + return self.type.marshal(v) + else: + return self.type(v) + + return_value = [] + try: + for item in value: + return_value.append(marshal(item)) + except Exception as err: + raise ManifestError(f'Failed to convert {self.key}={value} to array: {err}') + + return return_value + + def unmarshal(self, value): + def unmarshal(v): + if isinstance(self.type, ManifestSchema.ManifestNode): + return self.type.unmarshal(v) + else: + return str(v) + return [unmarshal(item) for item in value] + + # TODO: add description for each field + SCHEMA = ManifestRoot('root', [ + ManifestField('version', Version.parse, Version(1, 0, 0)), + ManifestRoot('package', [ + ManifestField('version', Version.parse), + ManifestField('base-os-constraint', VersionConstraint.parse, VersionRange()), + ManifestArray('depends', PackageConstraint.parse), + ManifestArray('breaks', PackageConstraint.parse), + ManifestField('init-cfg', dict, dict()), + ManifestField('changelog', dict, dict()), + ManifestField('debug-dump', str, ''), + ]), + ManifestRoot('service', [ + ManifestField('name', str), + ManifestArray('requires', str), + ManifestArray('requisite', str), + ManifestArray('wanted-by', str), + ManifestArray('after', str), + ManifestArray('before', str), + ManifestArray('dependent', str), + ManifestArray('dependent-of', str), + ManifestField('post-start-action', str, ''), + ManifestField('pre-shutdown-action', str, ''), + ManifestField('asic-service', bool, False), + ManifestField('host-service', bool, True), + ManifestField('delayed', bool, False), + ]), + ManifestRoot('container', [ + ManifestField('privileged', bool, False), + ManifestArray('volumes', str), + ManifestArray('mounts', ManifestRoot('mounts', [ + ManifestField('source', str), + ManifestField('target', str), + ManifestField('type', str), + ])), + ManifestField('environment', dict, dict()), + ]), + ManifestArray('processes', ManifestRoot('processes', [ + ManifestField('critical', bool), + ManifestField('name', str), + ManifestField('command', str), + ])), + ManifestRoot('cli', [ + ManifestField('show', str, ''), + ManifestField('config', str, ''), + ManifestField('clear', str, '') + ]) + ]) + + +class Manifest(dict): + """ Manifest object. """ + + SCHEMA = ManifestSchema.SCHEMA + LABEL_NAME = 'com.azure.sonic.manifest' + + @classmethod + def marshal(cls, input_dict: dict): + return Manifest(cls.SCHEMA.marshal(input_dict)) + + def unmarshal(self) -> Dict: + return self.SCHEMA.unmarshal(self) diff --git a/src/sonic-package-manager/sonic_package_manager/manifest_resolver.py b/src/sonic-package-manager/sonic_package_manager/manifest_resolver.py new file mode 100644 index 00000000000..b3c63f07a8d --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/manifest_resolver.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +import json + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.errors import ManifestError +from sonic_package_manager.logger import log +from sonic_package_manager.manifest import Manifest + + +class ManifestResolver: + """ Resolve manifest for package. It might resolve the package + manifest from local docker image in case package is installed or + access Docker Registry if package is not installed.""" + + def __init__(self, docker, registry_resolver): + self.docker = docker + self.registry_resolver = registry_resolver + + def get_manifest(self, package_info: PackageEntry, ref: str) -> Manifest: + """ Gets the package manifest. + Args: + package_info: PackageEntry object + ref: Reference to resolve, either tag or digest + Returns: + Manifest object. + """ + name = package_info.name + version = package_info.version + installed = package_info.installed + + # If the package is installed or the requested + # reference is not None read it from remote. + if installed and str(version) == ref: + log.debug(f'requesting manifest from local for {name} {ref}') + manifest = self.get_manifest_local(package_info) + else: + log.debug(f'requesting manifest from remote for {name} {ref}') + manifest = self.get_manifest_remote(package_info, ref) + + log.debug(f'manifest for package {name} {ref}: {manifest}') + + return manifest + + def get_manifest_local(self, package: PackageEntry) -> Manifest: + """ Gets the package manifest from locally installed docker image. + Args: + package: PackageEntry object + Returns: + Package manifest + Raises: + ManifestError + """ + + repo = package.repository + version = str(package.version) + + labels = self.docker.labels(repo, version) + + try: + return self.get_manifest_from_labels(labels) + except (KeyError, TypeError, ValueError): + raise ManifestError(f'No manifest defined for {repo}:{version}') + + def get_manifest_remote(self, package: PackageEntry, ref: str) -> Manifest: + """ Gets the package manifest from remote registry. + Args: + package: PackageEntry object + ref: Reference to resolve, either tag or digest + Returns: + Package manifest + Raises: + ManifestError + """ + + repo = package.repository + registry = self.registry_resolver.get_registry_for(repo) + + manifest = registry.manifest(repo, ref) + digest = manifest['config']['digest'] + + blob = registry.blobs(repo, digest) + labels = blob['config']['Labels'] + + try: + return self.get_manifest_from_labels(labels) + except (KeyError, TypeError, ValueError): + raise ManifestError(f'No manifest defined for {repo}:{ref}') + + def get_manifest_from_labels(self, labels) -> Manifest: + manifest_dict = labels[Manifest.LABEL_NAME] + return Manifest.marshal(json.loads(manifest_dict)) diff --git a/src/sonic-package-manager/sonic_package_manager/package.py b/src/sonic-package-manager/sonic_package_manager/package.py new file mode 100644 index 00000000000..e6e54a87271 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/package.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +from dataclasses import dataclass + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.manifest import Manifest + + +@dataclass +class Package: + entry: PackageEntry + reference: str + manifest: Manifest + + @property + def name(self): return self.entry.name + + @property + def repository(self): return self.entry.repository + + @property + def installed(self): return self.entry.installed + + @property + def built_in(self): return self.entry.built_in + + @property + def version(self): return self.entry.version diff --git a/src/sonic-package-manager/sonic_package_manager/progress.py b/src/sonic-package-manager/sonic_package_manager/progress.py new file mode 100644 index 00000000000..c4d21cf8a98 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/progress.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +import enlighten + +BAR_FMT = '{desc}{desc_pad}{percentage:3.0f}%|{bar}| {count:{len_total}.2f}/{total:.2f}{unit_pad}{unit} ' + \ + '[{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]' + +COUNTER_FMT = '{desc}{desc_pad}{count:.1f} {unit}{unit_pad}' + \ + '[{elapsed}, {rate:.2f}{unit_pad}{unit}/s]{fill}' + + +class ProgressManager: + """ ProgressManager is used for creating progress multiple bars + which nicely interact with logging and prints. + """ + + def __init__(self): + self.manager = enlighten.get_manager() + self.pbars = {} + + def __enter__(self): + return self.manager.__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + return self.manager.__exit__(exc_type, exc_val, exc_tb) + + def new(self, id: str, *args, **kwargs): + """ Creates new progress bar with id. + Args: + id: progress bar identifier + *args: pass arguments for progress bar creation + **kwargs: pass keyword arguments for progress bar creation. + """ + + if 'bar_format' not in kwargs: + kwargs['bar_format'] = BAR_FMT + if 'counter_format' not in kwargs: + kwargs['counter_format'] = COUNTER_FMT + + self.pbars[id] = self.manager.counter(*args, **kwargs) + + def get(self, id: str): + """ Returns progress bar by id. + Args: + id: progress bar identifier + Returns: + Progress bar. + """ + + return self.pbars[id] + + def __contains__(self, id): + return id in self.pbars diff --git a/src/sonic-package-manager/sonic_package_manager/reference.py b/src/sonic-package-manager/sonic_package_manager/reference.py new file mode 100644 index 00000000000..9c4d8e825c6 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/reference.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import re +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class PackageReference: + """ PackageReference is a package version constraint. """ + + name: str + reference: Optional[str] = None + + def __str__(self): + return f'{self.name} {self.reference}' + + @staticmethod + def parse(expression: str) -> 'PackageReference': + REQUIREMENT_SPECIFIER_RE = \ + r'(?P[A-Za-z0-9_-]+)(?P@(?P.*))' + + match = re.match(REQUIREMENT_SPECIFIER_RE, expression) + if match is None: + raise ValueError(f'Invalid reference specifier {expression}') + groupdict = match.groupdict() + name = groupdict.get('name') + reference = groupdict.get('reference') + + return PackageReference(name, reference) diff --git a/src/sonic-package-manager/sonic_package_manager/registry.py b/src/sonic-package-manager/sonic_package_manager/registry.py new file mode 100644 index 00000000000..e96259f512b --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/registry.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +import json +import time +from dataclasses import dataclass +from typing import List, Dict + +import requests +from docker_image import reference + +from sonic_package_manager.errors import PackageManagerError +from sonic_package_manager.logger import log + + +class AuthService: + """ AuthService provides an authentication tokens to access + Docker registry. """ + + @dataclass + class Token: + value: str + expires: int + + def __init__(self, host, service): + self._host = host + self._service = service + self._tokens = {} + + def get_token(self, repository: str, action: str = 'pull') -> str: + """ Retrieve an authentication token. If a token was previously + requested and did not expire yet it will be returned from the cache. + + Args: + repository: Repository to take action on. + action: Action to perform. + Returns: + token value as a string. + """ + + log.debug(f'getting authentication token for {repository}:{action}') + + scope = f'scope=repository:{repository}:{action}' + + # Try to get the token from cached tokens if cached token did not expire + if scope in self._tokens and self._tokens[scope].expires > time.time(): + token = self._tokens[scope] + log.debug(f'using cached authentication token for {repository}:{action}: {token}') + return token.value + + response = requests.get(f'{self._host}/token?{scope}&service={self._service}') + if response.status_code != requests.codes.ok: + raise PackageManagerError(f'Failed to retrieve token for {repository} action {action}') + + content = json.loads(response.content) + token = AuthService.Token(content['token'], time.time() + content['expires_in']) + + log.debug(f'authentication token for {repository}:{action}: {token}') + self._tokens[scope] = token + return token.value + + +class Registry: + """ Provides a Docker registry interface. """ + + MIME_DOCKER_MANIFEST = 'application/vnd.docker.distribution.manifest.v2+json' + + def __init__(self, host: str, auth=None): + self.url = host + self.auth = auth + + log.debug(f'initialized registry object with {self.url} {self.auth}') + + def _get_headers(self, repository: str): + _, repository = reference.Reference.split_docker_domain(repository) + headers = {} + if self.auth: + token = self.auth.get_token(repository) + headers['Authorization'] = f'Bearer {token}' + return headers + + def _get_base_url(self, repository: str): + return f'{self.url}/v2/{repository}' + + def tags(self, repository: str) -> List[str]: + log.debug(f'getting tags for {repository}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = self._get_headers(repository) + headers['Accept'] = 'application/json' + response = requests.get(f'{self._get_base_url(repository)}/tags/list', + headers=self._get_headers(repository)) + if response.status_code != requests.codes.ok: + raise PackageManagerError(f'Failed to retrieve tags from {repository}: ' + f'response: {response.status_code}, content: {response.content}') + + content = json.loads(response.content) + log.debug(f'tags list api response: f{content}') + + return content['tags'] + + def manifest(self, repository: str, ref: str) -> Dict: + log.debug(f'getting manifest for {repository}:{ref}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = self._get_headers(repository) + headers['Accept'] = self.MIME_DOCKER_MANIFEST + response = requests.get(f'{self._get_base_url(repository)}/manifests/{ref}', + headers=headers) + + if response.status_code != requests.codes.ok: + raise PackageManagerError(f'Failed to retrieve manifest for {repository}:{ref}: ' + f'response: {response.status_code}, content: {response.content}') + + content = json.loads(response.content) + log.debug(f'manifest content for {repository}:{ref}: {content}') + + return content + + def blobs(self, repository: str, digest: str): + log.debug(f'retrieving blob for {repository}:{digest}') + + _, repository = reference.Reference.split_docker_domain(repository) + headers = self._get_headers(repository) + headers['Accept'] = self.MIME_DOCKER_MANIFEST + response = requests.get(f'{self._get_base_url(repository)}/blobs/{digest}', + headers=headers) + if response.status_code != requests.codes.ok: + raise PackageManagerError(f'Failed to retrieve blobs for {repository}:{digest}: ' + f'response: {response.status_code}, content: {response.content}') + content = json.loads(response.content) + + log.debug(f'retrieved blob for {repository}:{digest}: {content}') + return content + + +class RegistryResolver: + """ Returns a registry object based on the input repository reference + string. """ + + DockerHubAuthService = AuthService('https://auth.docker.io', 'registry.docker.io') + DockerHubRegistry = Registry('https://index.docker.io', DockerHubAuthService) + + def __init__(self): + pass + + def get_registry_for(self, ref: str) -> Registry: + domain, _ = reference.Reference.split_docker_domain(ref) + if domain == reference.DEFAULT_DOMAIN: + return self.DockerHubRegistry + # TODO: authentication service for private registries + # TODO: support insecure registries + return Registry(f'https://{domain}') diff --git a/src/sonic-package-manager/sonic_package_manager/service_creator/__init__.py b/src/sonic-package-manager/sonic_package_manager/service_creator/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/sonic-package-manager/sonic_package_manager/service_creator/creator.py b/src/sonic-package-manager/sonic_package_manager/service_creator/creator.py new file mode 100644 index 00000000000..67fee176bcf --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/service_creator/creator.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python +import contextlib +import os +import stat +import subprocess +from typing import Dict + +import jinja2 as jinja2 + +from sonic_package_manager.logger import log +from sonic_package_manager.package import Package +from sonic_package_manager.service_creator.feature import FeatureRegistry +from sonic_package_manager.service_creator.utils import in_chroot + +SERVICE_FILE_TEMPLATE = 'sonic.service.j2' +TIMER_UNIT_TEMPLATE = 'timer.unit.j2' + +SYSTEMD_LOCATION = '/usr/lib/systemd/system' + +SERVICE_MGMT_SCRIPT_TEMPLATE = 'service_mgmt.sh.j2' +SERVICE_MGMT_SCRIPT_LOCATION = '/usr/local/bin' + +DOCKER_CTL_SCRIPT_TEMPLATE = 'docker_image_ctl.j2' +DOCKER_CTL_SCRIPT_LOCATION = '/usr/bin' + +MONIT_CONF_TEMPLATE = 'monit.conf.j2' +MONIT_CONF_LOCATION = '/etc/monit/conf.d/' + +DEBUG_DUMP_SCRIPT_TEMPLATE = 'dump.sh.j2' +DEBUG_DUMP_SCRIPT_LOCATION = '/usr/bin/debug-dump/' + +ETC_SONIC_PATH = '/etc/sonic' + +TEMPLATES_PATH = '/usr/share/sonic/templates' + + +class ServiceCreatorError(Exception): + pass + + +def render_template(in_template: str, + outfile: str, + render_ctx: Dict, + executable: bool = False): + """ Template renderer helper routine. + Args: + in_template: Input file with template content + outfile: Output file to render template to + render_ctx: Dictionary used to generate jinja2 template + executable: Set executable bit on rendered file + """ + + log.debug(f'Rendering {in_template} to {outfile} with {render_ctx}') + + with open(in_template, 'r') as instream: + template = jinja2.Template(instream.read()) + + with open(outfile, 'w') as outstream: + outstream.write(template.render(**render_ctx)) + + if executable: + set_executable_bit(outfile) + + +def get_template(template_name: str) -> str: + """ Returns a path to a template. + Args: + template_name: Template file name. + """ + + return os.path.join(TEMPLATES_PATH, template_name) + + +def set_executable_bit(filepath): + """ Sets +x on filepath. """ + + st = os.stat(filepath) + os.chmod(filepath, st.st_mode | stat.S_IEXEC) + + +def run_command(command: str): + """ Run arbitrary bash command. + Args: + command: String command to execute as bash script + Raises: + PackageManagerError: Raised when the command return code + is not 0. + """ + + log.info(f'running command: {command}') + + proc = subprocess.Popen(command, + shell=True, + executable='/bin/bash', + stdout=subprocess.PIPE) + (out, _) = proc.communicate() + if proc.returncode != 0: + raise ServiceCreatorError(f'Failed to execute "{command}"') + + +class ServiceCreator: + """ Creates and registers services in SONiC based on the package + manifest. """ + + def __init__(self, feature_registry: FeatureRegistry): + self.feature_registry = feature_registry + + def create(self, package: Package, register_feature=True): + try: + self.generate_container_mgmt(package) + self.generate_service_mgmt(package) + self.update_dependent_list_file(package) + self.generate_systemd_service(package) + self.generate_monit_conf(package) + self.generate_dump_script(package) + + if not in_chroot(): + run_command('systemctl daemon-reload') + run_command('systemctl reload monit') + + if register_feature: + self.feature_registry.register(package.name, package.manifest) + except Exception: + self.remove(package, not register_feature) + raise + except KeyboardInterrupt: + self.remove(package, not register_feature) + + def remove(self, package: Package, deregister_feature=True): + name = package.manifest['service']['name'] + + def remove_file(path): + if os.path.exists(path): + os.remove(path) + log.info(f'removed {path}') + + remove_file(os.path.join(MONIT_CONF_LOCATION, f'monit_{name}')) + remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}.service')) + remove_file(os.path.join(SYSTEMD_LOCATION, f'{name}@.service')) + remove_file(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh')) + remove_file(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh')) + remove_file(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}')) + + self.update_dependent_list_file(package, remove=True) + + if not in_chroot(): + run_command('systemctl daemon-reload') + run_command('systemctl reload monit') + + if deregister_feature: + self.feature_registry.deregister(package.name) + + def generate_container_mgmt(self, package: Package): + repository = package.repository + name = package.manifest['service']['name'] + container_spec = package.manifest['container'] + script_path = os.path.join(DOCKER_CTL_SCRIPT_LOCATION, f'{name}.sh') + script_template = os.path.join(TEMPLATES_PATH, DOCKER_CTL_SCRIPT_TEMPLATE) + run_opt = [] + + if container_spec['privileged']: + run_opt.append('--privileged') + + run_opt.append('-t') + + for volume in container_spec['volumes']: + run_opt.append(f'-v {volume}') + + for mount in container_spec['mounts']: + mount_type, source, target = mount['type'], mount['source'], mount['target'] + run_opt.append(f'--mount type={mount_type},source={source},target={target}') + + for env_name, value in container_spec['environment'].items(): + run_opt.append(f'-e {env_name}={value}') + + run_opt = ' '.join(run_opt) + render_ctx = { + 'docker_container_name': name, + 'docker_image_name': repository, + 'docker_image_run_opt': run_opt, + } + render_template(script_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') + + def generate_service_mgmt(self, package: Package): + name = package.manifest['service']['name'] + multi_instance_services = self.feature_registry.get_multi_instance_features() + scrip_template = os.path.join(TEMPLATES_PATH, SERVICE_MGMT_SCRIPT_TEMPLATE) + script_path = os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, f'{name}.sh') + render_ctx = { + 'source': get_template(SERVICE_MGMT_SCRIPT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance_services': multi_instance_services, + } + render_template(scrip_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') + + def generate_systemd_service(self, package: Package): + name = package.manifest['service']['name'] + multi_instance_services = self.feature_registry.get_multi_instance_features() + + template = os.path.join(TEMPLATES_PATH, SERVICE_FILE_TEMPLATE) + template_vars = { + 'source': get_template(SERVICE_FILE_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance': False, + 'multi_instance_services': multi_instance_services, + } + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}.service') + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['asic-service']: + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}@.service') + template_vars['multi_instance'] = True + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['delayed']: + template_vars = { + 'source': get_template(TIMER_UNIT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + 'multi_instance': False, + } + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}.timer') + template = os.path.join(TEMPLATES_PATH, TIMER_UNIT_TEMPLATE) + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + if package.manifest['service']['asic-service']: + output_file = os.path.join(SYSTEMD_LOCATION, f'{name}@.timer') + template_vars['multi_instance'] = True + render_template(template, output_file, template_vars) + log.info(f'generated {output_file}') + + def generate_monit_conf(self, package: Package): + name = package.manifest['service']['name'] + processes = package.manifest['processes'] + output_filename = os.path.join(MONIT_CONF_LOCATION, f'monit_{name}') + render_template(get_template(MONIT_CONF_TEMPLATE), output_filename, + {'source': get_template(MONIT_CONF_TEMPLATE), + 'feature': name, + 'processes': processes}) + log.info(f'generated {output_filename}') + + def update_dependent_list_file(self, package: Package, remove=False): + name = package.manifest['service']['name'] + dependent_of = package.manifest['service']['dependent-of'] + host_service = package.manifest['service']['host-service'] + asic_service = package.manifest['service']['asic-service'] + + def update_dependent(service, name, multi_inst): + if multi_inst: + filename = f'{service}_multi_inst_dependent' + else: + filename = f'{service}_dependent' + + filepath = os.path.join(ETC_SONIC_PATH, filename) + + dependent_services = set() + if os.path.exists(filepath): + with open(filepath) as fp: + dependent_services.update({line.strip() for line in fp.readlines()}) + if remove: + with contextlib.suppress(KeyError): + dependent_services.remove(name) + else: + dependent_services.add(name) + with open(filepath, 'w') as fp: + fp.write('\n'.join(dependent_services)) + + for service in dependent_of: + if host_service: + update_dependent(service, name, multi_inst=False) + if asic_service: + update_dependent(service, name, multi_inst=True) + + def generate_dump_script(self, package): + name = package.manifest['service']['name'] + + if not package.manifest['package']['debug-dump']: + return + + if not os.path.exists(DEBUG_DUMP_SCRIPT_LOCATION): + os.mkdir(DEBUG_DUMP_SCRIPT_LOCATION) + + scrip_template = os.path.join(TEMPLATES_PATH, DEBUG_DUMP_SCRIPT_TEMPLATE) + script_path = os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, f'{name}') + render_ctx = { + 'source': get_template(SERVICE_MGMT_SCRIPT_TEMPLATE), + 'manifest': package.manifest.unmarshal(), + } + render_template(scrip_template, script_path, render_ctx, executable=True) + log.info(f'generated {script_path}') diff --git a/src/sonic-package-manager/sonic_package_manager/service_creator/feature.py b/src/sonic-package-manager/sonic_package_manager/service_creator/feature.py new file mode 100644 index 00000000000..d6c6de67841 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/service_creator/feature.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python + +""" This module implements new feature registration/de-registration in SONiC system. """ + +from typing import Dict, Type + +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.service_creator.sonic_db import SonicDB + +FEATURE = 'FEATURE' + + +class FeatureRegistry: + """ FeatureRegistry class provides an interface to + register/de-register new feature persistently. """ + + def __init__(self, sonic_db: Type[SonicDB]): + self._sonic_db = sonic_db + + def register(self, name: str, manifest: Manifest): + for table in self._get_tables(): + cfg_entries = self.get_default_feature_entries() + non_cfg_entries = self.get_non_configurable_feature_entries(manifest) + + exists, running_cfg = table.get(name) + + cfg = cfg_entries.copy() + # Override configurable entries with CONFIG DB data. + cfg = {**cfg, **dict(running_cfg)} + # Override CONFIG DB data with non configurable entries. + cfg = {**cfg, **non_cfg_entries} + + table.set(name, list(cfg.items())) + + def deregister(self, name: str): + for table in self._get_tables(): + table._del(name) + + def is_feature_enabled(self, name: str) -> bool: + """ Returns whether the feature is current enabled + or not. Accesses running CONFIG DB. If no running CONFIG_DB + table is found in tables returns False. """ + + running_db_table = self._sonic_db.running_table(FEATURE) + if running_db_table is None: + return False + + exists, cfg = running_db_table.get(name) + if not exists: + return False + cfg = dict(cfg) + if cfg.get('state') == 'enabled': + return True + + return False + + def get_multi_instance_features(self): + res = [] + init_db_table = self._sonic_db.initial_table(FEATURE) + for feature in init_db_table.keys(): + exists, cfg = init_db_table.get(feature) + assert exists + cfg = dict(cfg) + asic_flag = str(cfg.get('has_per_asic_scope', 'False')) + if asic_flag.lower() == 'true': + res.append(feature) + return res + + @staticmethod + def get_default_feature_entries() -> Dict[str, str]: + """ Get configurable feature table entries: + e.g. 'state', 'auto_restart', etc. """ + + return { + 'state': 'disabled', + 'auto_restart': 'enabled', + 'high_mem_alert': 'disabled', + } + + @staticmethod + def get_non_configurable_feature_entries(manifest) -> Dict[str, str]: + """ Get non-configurable feature table entries: e.g. 'has_timer' """ + + return { + 'has_per_asic_scope': str(manifest['service']['asic-service']), + 'has_global_scope': str(manifest['service']['host-service']), + 'has_timer': str(manifest['service']['delayed']), + } + + def _get_tables(self): + tables = [] + running = self._sonic_db.running_table(FEATURE) + if running is not None: # it's Ok if there is no database + tables.append(running) + persistent = self._sonic_db.persistent_table(FEATURE) + if persistent is not None: # this is also Ok + tables.append(persistent) + tables.append(self._sonic_db.initial_table(FEATURE)) # init_cfg.json is must + + return tables diff --git a/src/sonic-package-manager/sonic_package_manager/service_creator/sonic_db.py b/src/sonic-package-manager/sonic_package_manager/service_creator/sonic_db.py new file mode 100644 index 00000000000..bf1654b453d --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/service_creator/sonic_db.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +import contextlib +import json +import os + +import swsssdk +from swsscommon import swsscommon + +from sonic_package_manager.service_creator.utils import in_chroot + +CONFIG_DB = 'CONFIG_DB' +CONFIG_DB_JSON = '/etc/sonic/config_db.json' +INIT_CFG_JSON = '/etc/sonic/init_cfg.json' + + +def is_db_alive(): + """ Check if database is alive or not. """ + + # In chroot we can connect to a running + # DB via TCP socket, we should ignore this case. + if in_chroot(): + return False + + # FIXME: swsscommon immediately crashes the whole python process + # in case redis connection failed. This is not possible to handle + # but installing a package while database is not running is completely + # OK for this application. Requires a fix in pyswsscommon + conn = swsssdk.ConfigDBConnector() + try: + conn.connect() + return True + except Exception: + return False + + +class FileDbTable: + """ swsscommon.Table adapter for persistent DBs. """ + + def __init__(self, file, table): + self._file = file + self._table = table + + def keys(self): + with open(self._file) as stream: + config = json.load(stream) + return config.get(self._table, {}).keys() + + def get(self, key): + with open(self._file) as stream: + config = json.load(stream) + + table = config.get(self._table, {}) + exists = key in table + fvs = table.get(key, {}) + return exists, list(fvs.items()) + + def set(self, key, fvs): + with open(self._file) as stream: + config = json.load(stream) + + table = config.setdefault(self._table, {}) + table.update({key: dict(fvs)}) + + with open(self._file, 'w') as stream: + json.dump(config, stream, indent=4) + + def _del(self, key): + with open(self._file) as stream: + config = json.load(stream) + + with contextlib.suppress(KeyError): + config[self._table].pop(key) + + with open(self._file, 'w') as stream: + json.dump(config, stream, indent=4) + + +class SonicDB: + """ Store different DB access objects for + running DB and also for persistent and initial + configs. """ + + _running = None + + @classmethod + def running_table(cls, table): + """ Returns running DB table. """ + + if not is_db_alive(): + return + + if cls._running is None: + cls._running = swsscommon.DBConnector(CONFIG_DB, 0) + + return swsscommon.Table(cls._running, table) + + @classmethod + def persistent_table(cls, table): + """ Returns persistent DB table. """ + + if not os.path.exists(CONFIG_DB_JSON): + return None + + return FileDbTable(CONFIG_DB_JSON, table) + + @classmethod + def initial_table(cls, table): + """ Returns initial DB table. """ + + return FileDbTable(INIT_CFG_JSON, table) diff --git a/src/sonic-package-manager/sonic_package_manager/service_creator/utils.py b/src/sonic-package-manager/sonic_package_manager/service_creator/utils.py new file mode 100644 index 00000000000..cdeeb17abb2 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/service_creator/utils.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python + +import os + + +def in_chroot() -> bool: + """ Verify if we are running in chroot or not + by comparing root / mount point device id and inode + with init process - /proc/1/root mount point device + id and inode. If those match we are not chroot-ed + otherwise we are. """ + + root_stat = os.stat('/') + init_root_stat = os.stat('/proc/1/root') + + return (root_stat.st_dev, root_stat.st_ino) != \ + (init_root_stat.st_dev, init_root_stat.st_ino) diff --git a/src/sonic-package-manager/sonic_package_manager/utils.py b/src/sonic-package-manager/sonic_package_manager/utils.py new file mode 100644 index 00000000000..95c6f718e61 --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/utils.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import keyword +import re + + +def make_python_identifier(string): + """ + Takes an arbitrary string and creates a valid Python identifier. + + Identifiers must follow the convention outlined here: + https://docs.python.org/2/reference/lexical_analysis.html#identifiers + """ + + # create a working copy (and make it lowercase, while we're at it) + s = string.lower() + + # remove leading and trailing whitespace + s = s.strip() + + # Make spaces into underscores + s = re.sub('[\\s\\t\\n]+', '_', s) + + # Remove invalid characters + s = re.sub('[^0-9a-zA-Z_]', '', s) + + # Remove leading characters until we find a letter or underscore + s = re.sub('^[^a-zA-Z_]+', '', s) + + # Check that the string is not a python identifier + while s in keyword.kwlist: + if re.match(".*?_\d+$", s): + i = re.match(".*?_(\d+)$", s).groups()[0] + s = s.strip('_'+i) + '_'+str(int(i)+1) + else: + s += '_1' + + return s diff --git a/src/sonic-package-manager/sonic_package_manager/version.py b/src/sonic-package-manager/sonic_package_manager/version.py new file mode 100644 index 00000000000..e5a5623d3bd --- /dev/null +++ b/src/sonic-package-manager/sonic_package_manager/version.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +""" Version and helpers routines. """ + +import semver + +Version = semver.Version +VersionRange = semver.VersionRange + + +def version_to_tag(ver: Version) -> str: + """ Converts the version to Docker compliant tag string. """ + + return str(ver).replace('+', '_') + + +def tag_to_version(tag: str) -> Version: + """ Converts the version to Docker compliant tag string. """ + + try: + return Version.parse(tag.replace('_', '+')) + except ValueError as err: + raise ValueError(f'Failed to convert {tag} to version string: {err}') diff --git a/src/sonic-package-manager/tests/conftest.py b/src/sonic-package-manager/tests/conftest.py new file mode 100644 index 00000000000..490c83c1064 --- /dev/null +++ b/src/sonic-package-manager/tests/conftest.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python +import os +from unittest.mock import Mock, MagicMock + +import pytest + +from sonic_package_manager.database import PackageDatabase, PackageEntry +from sonic_package_manager.manager import DockerApi, PackageManager +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.manifest_resolver import ManifestResolver +from sonic_package_manager.registry import RegistryResolver +from sonic_package_manager.service_creator.creator import ( + TEMPLATES_PATH, SERVICE_FILE_TEMPLATE, TIMER_UNIT_TEMPLATE, + SERVICE_MGMT_SCRIPT_TEMPLATE, DOCKER_CTL_SCRIPT_TEMPLATE, MONIT_CONF_TEMPLATE, SERVICE_MGMT_SCRIPT_LOCATION, + SYSTEMD_LOCATION, ETC_SONIC_PATH, MONIT_CONF_LOCATION, DOCKER_CTL_SCRIPT_LOCATION, DEBUG_DUMP_SCRIPT_TEMPLATE +) +from sonic_package_manager.version import Version + + +@pytest.fixture +def mock_docker_api(): + yield MagicMock(DockerApi) + + +@pytest.fixture +def mock_registry_resolver(): + yield Mock(RegistryResolver) + + +@pytest.fixture +def mock_manifest_resolver(): + yield Mock(ManifestResolver) + + +@pytest.fixture +def mock_feature_registry(): + yield MagicMock() + + +@pytest.fixture +def fake_manifest_resolver(): + class FakeManifestResolver: + def __init__(self): + self.manifests = { + ('database', '1.0.0'): Manifest.marshal({ + 'package': { + 'version': '1.0.0', + }, + 'service': { + 'name': 'database', + }, + }), + ('swss', '1.0.0'): Manifest.marshal({ + 'package': { + 'version': '1.0.0', + }, + 'service': { + 'name': 'swss', + }, + }), + ('test-package', '1.6.0'): Manifest.marshal({ + 'package': { + 'version': '1.6.0', + }, + 'service': { + 'name': 'test-package', + }, + }), + ('test-package-2', '1.5.0'): Manifest.marshal({ + 'package': { + 'version': '1.5.0', + }, + 'service': { + 'name': 'test-package-2', + }, + }), + ('test-package-3', '1.5.0'): Manifest.marshal({ + 'package': { + 'version': '1.5.0', + }, + 'service': { + 'name': 'test-package-3', + }, + }), + ('test-package-3', '1.6.0'): Manifest.marshal({ + 'package': { + 'version': '1.6.0', + }, + 'service': { + 'name': 'test-package-3', + }, + }), + ('test-package-4', '1.5.0'): Manifest.marshal({ + 'package': { + 'version': '1.5.0', + }, + 'service': { + 'name': 'test-package-4', + }, + }), + ('test-package-5', '1.5.0'): Manifest.marshal({ + 'package': { + 'version': '1.5.0', + }, + 'service': { + 'name': 'test-package-5', + }, + }), + ('test-package-5', '1.9.0'): Manifest.marshal({ + 'package': { + 'version': '1.9.0', + }, + 'service': { + 'name': 'test-package-5', + }, + }), + ('test-package-6', '1.5.0'): Manifest.marshal({ + 'package': { + 'version': '1.5.0', + }, + 'service': { + 'name': 'test-package-6', + }, + }), + ('test-package-6', '2.0.0'): Manifest.marshal({ + 'package': { + 'version': '2.0.0', + }, + 'service': { + 'name': 'test-package-6', + }, + }), + } + + def get_manifest(self, package_info: PackageEntry, ref: str) -> Manifest: + return self.manifests[(package_info.name, ref)] + + yield FakeManifestResolver() + + +@pytest.fixture +def mock_service_creator(): + yield Mock() + + +@pytest.fixture +def fake_device_info(): + class FakeDeviceInfo: + def is_multi_npu(self): + return False + + def get_num_npus(self): + return 1 + + def get_sonic_version_info(self): + return { + 'base-os-compatibility-version': '1.0.0' + } + + yield FakeDeviceInfo() + + +@pytest.fixture +def mock_sonic_db(): + yield Mock() + + +@pytest.fixture +def package_manager(mock_docker_api, + mock_registry_resolver, + mock_service_creator, + fake_manifest_resolver, + fake_db, + fake_device_info): + yield PackageManager(mock_docker_api, mock_registry_resolver, + fake_db, fake_manifest_resolver, + mock_service_creator, + fake_device_info) + + +@pytest.fixture +def fake_db(mock_docker_api, mock_registry_resolver): + content = { + 'database': PackageEntry(name='database', + repository='docker-database', + description='SONiC database service', + default_reference='1.0.0', + version=Version(1, 0, 0), + installed=True, built_in=True), + 'swss': PackageEntry(name='swss', + repository='docker-orchagent', + description='SONiC switch state service', + default_reference='1.0.0', + version=Version(1, 0, 0), + installed=True, built_in=True), + 'test-package': PackageEntry(name='test-package', + repository='Azure/docker-test', + description='SONiC Package Manager Test Package', + default_reference='1.6.0', + installed=False, built_in=False), + 'test-package-2': PackageEntry(name='test-package-2', + repository='Azure/docker-test-2', + description='SONiC Package Manager Test Package #2', + default_reference='1.5.0', + installed=False, built_in=False), + 'test-package-3': PackageEntry(name='test-package-3', + repository='Azure/docker-test-3', + description='SONiC Package Manager Test Package #3', + default_reference='1.5.0', + version=Version(1, 5, 0), + installed=True, built_in=False), + 'test-package-5': PackageEntry(name='test-package-5', + repository='Azure/docker-test-5', + description='SONiC Package Manager Test Package #5', + default_reference='1.9.0', + version=Version(1, 9, 0), + installed=False, built_in=False), + 'test-package-6': PackageEntry(name='test-package-6', + repository='Azure/docker-test-6', + description='SONiC Package Manager Test Package #6', + default_reference='1.5.0', + installed=False, built_in=False), + } + + yield PackageDatabase(content) + + +@pytest.fixture +def fake_db_for_migration(mock_docker_api, mock_registry_resolver): + content = { + 'database': PackageEntry(name='database', + repository='docker-database', + description='SONiC database service', + default_reference='1.0.0', + version=Version(1, 0, 0), + installed=True, built_in=True), + 'swss': PackageEntry(name='swss', + repository='docker-orchagent', + description='SONiC switch state service', + default_reference='1.0.0', + version=Version(1, 0, 0), + installed=True, built_in=True), + 'test-package': PackageEntry(name='test-package', + repository='Azure/docker-test', + description='SONiC Package Manager Test Package', + default_reference='1.6.0', + installed=False, built_in=False), + 'test-package-2': PackageEntry(name='test-package-2', + repository='Azure/docker-test-2', + description='SONiC Package Manager Test Package #2', + default_reference='2.0.0', + installed=False, built_in=False), + 'test-package-3': PackageEntry(name='test-package-3', + repository='Azure/docker-test-3', + description='SONiC Package Manager Test Package #2', + default_reference='1.6.0', + version=Version(1, 6, 0), + installed=True, built_in=False), + 'test-package-4': PackageEntry(name='test-package-4', + repository='Azure/docker-test-4', + description='SONiC Package Manager Test Package #4', + default_reference='1.5.0', + version=Version(1, 5, 0), + installed=True, built_in=False), + 'test-package-5': PackageEntry(name='test-package-5', + repository='Azure/docker-test-5', + description='SONiC Package Manager Test Package #5', + default_reference='1.5.0', + version=Version(1, 5, 0), + installed=True, built_in=False), + 'test-package-6': PackageEntry(name='test-package-6', + repository='Azure/docker-test-6', + description='SONiC Package Manager Test Package #6', + default_reference='1.5.0', + version=Version(2, 0, 0), + installed=True, built_in=False), + } + + yield PackageDatabase(content) + + +@pytest.fixture +def sonic_fs(fs): + fs.create_file('/proc/1/root') + fs.create_dir(ETC_SONIC_PATH) + fs.create_dir(SYSTEMD_LOCATION) + fs.create_dir(DOCKER_CTL_SCRIPT_LOCATION) + fs.create_dir(SERVICE_MGMT_SCRIPT_LOCATION) + fs.create_dir(MONIT_CONF_LOCATION) + fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_FILE_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, TIMER_UNIT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, SERVICE_MGMT_SCRIPT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, DOCKER_CTL_SCRIPT_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, MONIT_CONF_TEMPLATE)) + fs.create_file(os.path.join(TEMPLATES_PATH, DEBUG_DUMP_SCRIPT_TEMPLATE)) + yield fs + + +@pytest.fixture +def anything(): + """ Fixture that returns Any object that can be used in + assert_called_*_with to match any object passed. """ + + class Any: + def __eq__(self, other): + return True + + yield Any() diff --git a/src/sonic-package-manager/tests/test_cli.py b/src/sonic-package-manager/tests/test_cli.py new file mode 100644 index 00000000000..ae168d125b4 --- /dev/null +++ b/src/sonic-package-manager/tests/test_cli.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +from click.testing import CliRunner + +from sonic_package_manager import main + + +def test_show_changelog(package_manager, fake_manifest_resolver): + """ Test case for "sonic-package-manager package show changelog [NAME]" """ + + runner = CliRunner() + changelog = { + "1.0.0": { + "changes": ["Initial release"], + "author": "Stepan Blyshchak", + "email": "stepanb@nvidia.com", + "date": "Mon, 25 May 2020 12:24:30 +0300" + }, + "1.1.0": { + "changes": [ + "Added functionality", + "Bug fixes" + ], + "author": "Stepan Blyshchak", + "email": "stepanb@nvidia.com", + "date": "Fri, 23 Oct 2020 12:26:08 +0300" + } + } + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['changelog'] = changelog + + expected_output = """\ +1.0.0: + + • Initial release + + Stepan Blyshchak (stepanb@nvidia.com) Mon, 25 May 2020 12:24:30 +0300 + +1.1.0: + + • Added functionality + • Bug fixes + + Stepan Blyshchak (stepanb@nvidia.com) Fri, 23 Oct 2020 12:26:08 +0300 + +""" + + result = runner.invoke(main.package.commands['show'].commands['changelog'], ['test-package'], obj=package_manager) + + assert result.exit_code == 0 + assert result.output == expected_output + + +def test_show_changelog_no_changelog(package_manager): + """ Test case for "sonic-package-manager package show changelog [NAME]" + when there is no changelog provided by package. """ + + runner = CliRunner() + result = runner.invoke(main.package.commands['show'].commands['changelog'], ['test-package'], obj=package_manager) + + assert result.exit_code == 1 + assert result.output == 'Failed to print package changelog: No changelog for package test-package\n' diff --git a/src/sonic-package-manager/tests/test_constraint.py b/src/sonic-package-manager/tests/test_constraint.py new file mode 100644 index 00000000000..d9f942d330f --- /dev/null +++ b/src/sonic-package-manager/tests/test_constraint.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +from sonic_package_manager import version +from sonic_package_manager.constraint import PackageConstraint +from sonic_package_manager.version import Version, VersionRange + + +def test_constraint(): + package_constraint = PackageConstraint.parse('swss>1.0.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('0.9.1')) + assert package_constraint.constraint.allows(Version.parse('1.1.1')) + + +def test_constraint_range(): + package_constraint = PackageConstraint.parse('swss^1.2.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.5')) + assert not package_constraint.constraint.allows(Version.parse('2.0.1')) + + +def test_constraint_strict(): + package_constraint = PackageConstraint.parse('swss==1.2.0') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.0')) + + +def test_constraint_match(): + package_constraint = PackageConstraint.parse('swss==1.2*.*') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('1.1.1')) + assert package_constraint.constraint.allows(Version.parse('1.2.0')) + + +def test_constraint_multiple(): + package_constraint = PackageConstraint.parse('swss>1.2.0,<3.0.0,!=2.2.2') + assert package_constraint.name == 'swss' + assert not package_constraint.constraint.allows(Version.parse('2.2.2')) + assert not package_constraint.constraint.allows(Version.parse('3.2.0')) + assert not package_constraint.constraint.allows(Version.parse('0.2.0')) + assert package_constraint.constraint.allows(Version.parse('2.2.3')) + assert package_constraint.constraint.allows(Version.parse('1.2.3')) + + +def test_constraint_only_name(): + package_constraint = PackageConstraint.parse('swss') + assert package_constraint.name == 'swss' + assert package_constraint.constraint == VersionRange() + + +def test_version_to_tag(): + assert version.version_to_tag(Version.parse('1.0.0-rc0')) == '1.0.0-rc0' + assert version.version_to_tag(Version.parse('1.0.0-rc0+152')) == '1.0.0-rc0_152' + + +def test_tag_to_version(): + assert str(version.tag_to_version('1.0.0-rc0_152')) == '1.0.0-rc0+152' + assert str(version.tag_to_version('1.0.0-rc0')) == '1.0.0-rc0' diff --git a/src/sonic-package-manager/tests/test_database.py b/src/sonic-package-manager/tests/test_database.py new file mode 100644 index 00000000000..7e9867080d8 --- /dev/null +++ b/src/sonic-package-manager/tests/test_database.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +from unittest.mock import Mock + +import pytest + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.errors import ( + PackageNotFoundError, + PackageAlreadyExistsError, + PackageManagerError +) +from sonic_package_manager.version import Version + + +def test_database_get_package(fake_db): + swss_package = fake_db.get_package('swss') + assert swss_package.installed + assert swss_package.built_in + assert swss_package.repository == 'docker-orchagent' + assert swss_package.default_reference == '1.0.0' + assert swss_package.version == Version(1, 0, 0) + + +def test_database_get_package_not_essential(fake_db): + test_package = fake_db.get_package('test-package') + assert not test_package.installed + assert not test_package.built_in + assert test_package.repository == 'Azure/docker-test' + assert test_package.default_reference == '1.6.0' + assert test_package.version is None + + +def test_database_get_package_not_existing(fake_db): + with pytest.raises(PackageNotFoundError): + fake_db.get_package('abc') + + +def test_database_add_package(fake_db): + fake_db.add_package('test-package-99', 'Azure/docker-test-99') + test_package = fake_db.get_package('test-package-99') + assert not test_package.installed + assert not test_package.built_in + assert test_package.repository == 'Azure/docker-test-99' + assert test_package.default_reference is None + assert test_package.version is None + + +def test_database_add_package_existing(fake_db): + with pytest.raises(PackageAlreadyExistsError): + fake_db.add_package('swss', 'Azure/docker-orchagent') + + +def test_database_update_package(fake_db): + test_package = fake_db.get_package('test-package-2') + test_package.installed = True + test_package.version = Version(1, 2, 3) + fake_db.update_package(test_package) + test_package = fake_db.get_package('test-package-2') + assert test_package.installed + assert test_package.version == Version(1, 2, 3) + + +def test_database_update_package_non_existing(fake_db): + test_package = PackageEntry('abc', 'abc') + with pytest.raises(PackageNotFoundError): + fake_db.update_package(test_package) + + +def test_database_remove_package(fake_db): + fake_db.remove_package('test-package') + assert not fake_db.has_package('test-package') + + +def test_database_remove_package_non_existing(fake_db): + with pytest.raises(PackageNotFoundError): + fake_db.remove_package('non-existing-package') + + +def test_database_remove_package_installed(fake_db): + with pytest.raises(PackageManagerError, + match='Package test-package-3 is installed, ' + 'uninstall it first'): + fake_db.remove_package('test-package-3') + + +def test_database_remove_package_built_in(fake_db): + with pytest.raises(PackageManagerError, + match='Package swss is built-in, ' + 'cannot remove it'): + fake_db.remove_package('swss') diff --git a/src/sonic-package-manager/tests/test_manager.py b/src/sonic-package-manager/tests/test_manager.py new file mode 100644 index 00000000000..607af275b7b --- /dev/null +++ b/src/sonic-package-manager/tests/test_manager.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +from unittest.mock import Mock, call + +import pytest + +from sonic_package_manager.constraint import PackageConstraint +from sonic_package_manager.errors import ( + PackageInstallationError, + PackageSonicRequirementError, + PackageManagerError +) +from sonic_package_manager.version import Version, VersionRange + + +def test_installation_not_installed(package_manager): + package_manager.install('test-package') + + +def test_installation_already_installed(package_manager): + with pytest.raises(PackageInstallationError, + match='swss is already installed'): + package_manager.install('swss') + + +def test_installation_dependencies(package_manager, fake_manifest_resolver): + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['depends'] = [PackageConstraint.parse('swss^2.0.0')] + with pytest.raises(PackageInstallationError, + match='Package test-package requires ' + 'swss>=2.0.0,<3.0.0 but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_dependencies_missing_package(package_manager, fake_manifest_resolver): + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['depends'] = [PackageConstraint.parse('missing-package>=1.0.0')] + with pytest.raises(PackageInstallationError, + match='Package test-package requires ' + 'missing-package>=1.0.0 but it is not installed'): + package_manager.install('test-package') + + +def test_installation_dependencies_satisfied(package_manager, fake_manifest_resolver): + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['depends'] = [ + PackageConstraint.parse('database>=1.0.0'), + PackageConstraint.parse('swss>=1.0.0'), + ] + package_manager.install('test-package') + + +def test_installation_breaks(package_manager, fake_manifest_resolver): + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['breaks'] = [PackageConstraint.parse('swss^1.0.0')] + with pytest.raises(PackageInstallationError, + match='Package test-package conflicts with ' + 'swss>=1.0.0,<2.0.0 but version 1.0.0 is installed'): + package_manager.install('test-package') + + +def test_installation_breaks_missing_package(package_manager, fake_manifest_resolver): + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['breaks'] = [PackageConstraint.parse('missing-package^1.0.0')] + package_manager.install('test-package') + + +def test_installation_breaks_not_installed_package(package_manager, fake_manifest_resolver): + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['breaks'] = [PackageConstraint.parse('test-package-2^1.0.0')] + package_manager.install('test-package') + + +def test_installation_base_os_constraint(package_manager, fake_manifest_resolver): + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['base-os-constraint'] = VersionRange(min=Version(2, 0, 0), include_min=True) + with pytest.raises(PackageSonicRequirementError, + match='Package test-package requires ' + 'base OS compatibility version >=2.0.0 ' + 'while the installed version is 1.0.0'): + package_manager.install('test-package') + + +def test_installation_base_os_constraint_satisfied(package_manager, fake_manifest_resolver): + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + manifest['package']['base-os-constraint'] = VersionRange(min=Version(1, 0, 0), + max=Version(2, 0, 0), + include_min=True) + package_manager.install('test-package') + + +def test_installation(package_manager, mock_docker_api, anything): + package_manager.install('test-package') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', + '1.6.0', + Version.parse('1.6.0'), + anything) + mock_docker_api.tag.assert_called_once_with('Azure/docker-test', + Version.parse('1.6.0'), + 'Azure/docker-test', + 'latest') + + +def test_installation_using_reference(package_manager, + fake_manifest_resolver, + mock_docker_api, + anything): + ref = 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + manifest = fake_manifest_resolver.manifests[('test-package', '1.6.0')] + fake_manifest_resolver.manifests[('test-package', ref)] = manifest + + package_manager.install(f'test-package@{ref}') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', + f'{ref}', + Version.parse('1.6.0'), + anything) + mock_docker_api.tag.assert_called_once_with('Azure/docker-test', + Version.parse('1.6.0'), + 'Azure/docker-test', + 'latest') + + +def test_manager_installation_tag(package_manager, + mock_docker_api, + anything): + package_manager.install(f'test-package==1.6.0') + mock_docker_api.pull.assert_called_once_with('Azure/docker-test', + '1.6.0', + Version.parse('1.6.0'), + anything) + mock_docker_api.tag.assert_called_once_with('Azure/docker-test', + Version.parse('1.6.0'), + 'Azure/docker-test', + 'latest') + + +def test_manager_installation_version_range(package_manager): + with pytest.raises(PackageManagerError, + match='Can only install specific version. ' + 'Use only following expression "test-package==" ' + 'to install specific version'): + package_manager.install(f'test-package>=1.6.0') + + +def test_manager_migration(package_manager, fake_db_for_migration): + package_manager.install = Mock() + package_manager.upgrade = Mock() + package_manager.migrate_packages(fake_db_for_migration) + + # test-package-3 was installed but there is a newer version installed + # in fake_db_for_migration, asserting for upgrade + package_manager.upgrade.assert_has_calls([call('test-package-3==1.6.0')], any_order=True) + + package_manager.install.assert_has_calls([ + # test-package-4 was not present in DB at all, but it is present and installed in + # fake_db_for_migration, thus asserting that it is going to be installed. + call('test-package-4==1.5.0'), + # test-package-5 1.5.0 was installed in fake_db_for_migration but the default + # in current db is 1.9.0, assert that migration will install the newer version. + call('test-package-5==1.9.0'), + # test-package-6 2.0.0 was installed in fake_db_for_migration but the default + # in current db is 1.5.0, assert that migration will install the newer version. + call('test-package-6==2.0.0')], + any_order=True + ) diff --git a/src/sonic-package-manager/tests/test_manifest.py b/src/sonic-package-manager/tests/test_manifest.py new file mode 100644 index 00000000000..47faec0e4b9 --- /dev/null +++ b/src/sonic-package-manager/tests/test_manifest.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +import pytest + +from sonic_package_manager.manifest import Manifest, ManifestError +from sonic_package_manager.version import VersionRange + + +def test_manifest_v1_defaults(): + manifest = Manifest.marshal({'package': {'version': '1.0.0'}, 'service': {'name': 'test'}}) + assert manifest['package']['depends'] == [] + assert manifest['package']['breaks'] == [] + assert manifest['package']['base-os-constraint'] == VersionRange() + assert not manifest['service']['asic-service'] + assert manifest['service']['host-service'] + + +def test_manifest_v1_invalid_version(): + with pytest.raises(ManifestError): + Manifest.marshal({'package': {'version': 'abc'}, 'service': {'name': 'test'}}) + + +def test_manifest_v1_invalid_package_constraint(): + with pytest.raises(ManifestError): + Manifest.marshal({'package': {'version': '1.0.0', 'depends': ['swss>a']}, 'service': {'name': 'test'}}) + + +def test_manifest_v1_service_spec(): + manifest = Manifest.marshal({'package': {'version': '1.0.0'}, 'service': {'name': 'test', 'asic-service': True}}) + assert manifest['service']['asic-service'] + + +def test_manifest_v1_mounts(): + manifest = Manifest.marshal({'version': '1.0.0', 'package': {'version': '1.0.0'}, + 'service': {'name': 'cpu-report'}, + 'container': {'privileged': True, + 'mounts': [{'source': 'a', 'target': 'b', 'type': 'bind'}]}}) + assert manifest['container']['mounts'][0]['source'] == 'a' + assert manifest['container']['mounts'][0]['target'] == 'b' + assert manifest['container']['mounts'][0]['type'] == 'bind' + + +def test_manifest_v1_mounts_invalid(): + with pytest.raises(ManifestError): + Manifest.marshal({'version': '1.0.0', 'package': {'version': '1.0.0'}, + 'service': {'name': 'cpu-report'}, + 'container': {'privileged': True, + 'mounts': [{'not-source': 'a', 'target': 'b', 'type': 'bind'}]}}) + + +def test_manifest_v1_unmarshal(): + manifest_json_input = {'package': {'version': '1.0.0', + 'depends': ['swss>1.0.0']}, + 'service': {'name': 'test'}} + manifest = Manifest.marshal(manifest_json_input) + manifest_json = manifest.unmarshal() + for key, section in manifest_json_input.items(): + for field, value in section.items(): + assert manifest_json[key][field] == value diff --git a/src/sonic-package-manager/tests/test_manifest_resolver.py b/src/sonic-package-manager/tests/test_manifest_resolver.py new file mode 100644 index 00000000000..72b9f5943ea --- /dev/null +++ b/src/sonic-package-manager/tests/test_manifest_resolver.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +import contextlib +from unittest.mock import Mock, MagicMock + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.errors import ManifestError +from sonic_package_manager.manifest_resolver import ManifestResolver +from sonic_package_manager.version import Version + + +def test_manifest_resolver_local(mock_registry_resolver, mock_docker_api): + manifest_resolver = ManifestResolver(mock_docker_api, mock_registry_resolver) + package_entry = PackageEntry(name='test', + repository='test-repository', + installed=True, + version=Version(1, 2, 0)) + # it raises exception because mock manifest is not a valid manifest + # but this is not a test objective, so just suppress the error. + with contextlib.suppress(ManifestError): + manifest_resolver.get_manifest(package_entry, '1.2.0') + mock_docker_api.labels.assert_called_once_with('test-repository', '1.2.0') + + +def test_manifest_resolver_remote(mock_registry_resolver, mock_docker_api): + manifest_resolver = ManifestResolver(mock_docker_api, mock_registry_resolver) + mock_registry = MagicMock() + mock_registry.manifest = MagicMock(return_value={'config': {'digest': 'some-digest'}}) + package_entry = PackageEntry(name='test', + repository='test-repository', + installed=False) + + def return_mock_registry(repository): + return mock_registry + + mock_registry_resolver.get_registry_for = Mock(side_effect=return_mock_registry) + # it raises exception because mock manifest is not a valid manifest + # but this is not a test objective, so just suppress the error. + with contextlib.suppress(ManifestError): + manifest_resolver.get_manifest(package_entry, '1.2.0') + mock_registry_resolver.get_registry_for.assert_called_once_with('test-repository') + mock_registry.manifest.assert_called_once_with('test-repository', '1.2.0') + mock_registry.blobs.assert_called_once_with('test-repository', 'some-digest') + mock_docker_api.labels.assert_not_called() + + +def test_manifest_resolver_remote_different_tag(mock_registry_resolver, mock_docker_api): + manifest_resolver = ManifestResolver(mock_docker_api, mock_registry_resolver) + mock_registry = MagicMock() + mock_registry.manifest = MagicMock(return_value={'config': {'digest': 'some-digest'}}) + package_entry = PackageEntry(name='test', + repository='test-repository', + installed=True, + version=Version(1, 5, 0)) + + def return_mock_registry(repository): + return mock_registry + + mock_registry_resolver.get_registry_for = Mock(side_effect=return_mock_registry) + # it raises exception because mock manifest is not a valid manifest + # but this is not a test objective, so just suppress the error. + with contextlib.suppress(ManifestError): + manifest_resolver.get_manifest(package_entry, '1.2.0') + mock_registry_resolver.get_registry_for.assert_called_once_with('test-repository') + mock_registry.manifest.assert_called_once_with('test-repository', '1.2.0') + mock_registry.blobs.assert_called_once_with('test-repository', 'some-digest') + mock_docker_api.labels.assert_not_called() diff --git a/src/sonic-package-manager/tests/test_reference.py b/src/sonic-package-manager/tests/test_reference.py new file mode 100644 index 00000000000..c986632c43f --- /dev/null +++ b/src/sonic-package-manager/tests/test_reference.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import pytest + +from sonic_package_manager.reference import PackageReference + + +def test_reference(): + package_constraint = PackageReference.parse( + 'swss@sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + ) + assert package_constraint.name == 'swss' + assert package_constraint.reference == 'sha256:9780f6d83e45878749497a6297ed9906c19ee0cc48cc88dc63827564bb8768fd' + + +def test_reference_invalid(): + with pytest.raises(ValueError): + PackageReference.parse('swssfdsf') diff --git a/src/sonic-package-manager/tests/test_registry.py b/src/sonic-package-manager/tests/test_registry.py new file mode 100644 index 00000000000..0d82499df3d --- /dev/null +++ b/src/sonic-package-manager/tests/test_registry.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +from sonic_package_manager.registry import RegistryResolver + + +def test_get_registry_for(): + resolver = RegistryResolver() + registry = resolver.get_registry_for('debian') + assert registry is resolver.DockerHubRegistry + registry = resolver.get_registry_for('Azure/sonic') + assert registry is resolver.DockerHubRegistry + registry = resolver.get_registry_for('registry-server:5000/docker') + assert registry.url == 'https://registry-server:5000' + registry = resolver.get_registry_for('registry-server.com/docker') + assert registry.url == 'https://registry-server.com' diff --git a/src/sonic-package-manager/tests/test_service_creator.py b/src/sonic-package-manager/tests/test_service_creator.py new file mode 100644 index 00000000000..b4319a619b8 --- /dev/null +++ b/src/sonic-package-manager/tests/test_service_creator.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +import os +from unittest.mock import Mock + +import pytest + +from sonic_package_manager.database import PackageEntry +from sonic_package_manager.manifest import Manifest +from sonic_package_manager.package import Package +from sonic_package_manager.service_creator.creator import ( + ServiceCreator, + ETC_SONIC_PATH, DOCKER_CTL_SCRIPT_LOCATION, + SERVICE_MGMT_SCRIPT_LOCATION, SYSTEMD_LOCATION, MONIT_CONF_LOCATION, DEBUG_DUMP_SCRIPT_LOCATION +) +from sonic_package_manager.service_creator.feature import FeatureRegistry + + +@pytest.fixture +def manifest(): + return Manifest.marshal({ + 'package': { + 'name': 'test', + 'version': '1.0.0', + }, + 'service': { + 'name': 'test', + 'requires': ['database'], + 'after': ['database', 'swss', 'syncd'], + 'before': ['ntp-config'], + 'dependent-of': ['swss'], + 'asic-service': False, + 'host-service': True, + }, + 'container': { + 'privileged': True, + 'volumes': [ + '/etc/sonic:/etc/sonic:ro' + ] + } + }) + + +def test_service_creator(sonic_fs, manifest, mock_feature_registry): + creator = ServiceCreator(mock_feature_registry) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, '1.0.0', manifest) + creator.create(package) + + assert sonic_fs.exists(os.path.join(ETC_SONIC_PATH, 'swss_dependent')) + assert sonic_fs.exists(os.path.join(DOCKER_CTL_SCRIPT_LOCATION, 'test.sh')) + assert sonic_fs.exists(os.path.join(SERVICE_MGMT_SCRIPT_LOCATION, 'test.sh')) + assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.service')) + assert sonic_fs.exists(os.path.join(MONIT_CONF_LOCATION, 'monit_test')) + + +def test_service_creator_with_timer_unit(sonic_fs, manifest, mock_feature_registry): + creator = ServiceCreator(mock_feature_registry) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, '1.0.0', manifest) + creator.create(package) + + assert not sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) + + manifest['service']['delayed'] = True + package = Package(entry, '1.0.0', manifest) + creator.create(package) + + assert sonic_fs.exists(os.path.join(SYSTEMD_LOCATION, 'test.timer')) + + +def test_service_creator_with_debug_dump(sonic_fs, manifest, mock_feature_registry): + creator = ServiceCreator(mock_feature_registry) + entry = PackageEntry('test', 'azure/sonic-test') + package = Package(entry, '1.0.0', manifest) + creator.create(package) + + assert not sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + + manifest['package']['debug-dump'] = '/some/command' + package = Package(entry, '1.0.0', manifest) + creator.create(package) + + assert sonic_fs.exists(os.path.join(DEBUG_DUMP_SCRIPT_LOCATION, 'test')) + + +def test_feature_registration(mock_sonic_db, manifest): + mock_feature_table = Mock() + mock_feature_table.get = Mock(return_value=(False, ())) + mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) + mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + feature_registry = FeatureRegistry(mock_sonic_db) + feature_registry.register('test', manifest) + mock_feature_table.set.assert_called_with('test', [ + ('state', 'disabled'), + ('auto_restart', 'enabled'), + ('high_mem_alert', 'disabled'), + ('has_per_asic_scope', 'False'), + ('has_global_scope', 'True'), + ('has_timer', 'False') + ]) + + +def test_feature_registration_with_timer(mock_sonic_db, manifest): + manifest['service']['delayed'] = True + mock_feature_table = Mock() + mock_feature_table.get = Mock(return_value=(False, ())) + mock_sonic_db.initial_table = Mock(return_value=mock_feature_table) + mock_sonic_db.persistent_table = Mock(return_value=mock_feature_table) + mock_sonic_db.running_table = Mock(return_value=mock_feature_table) + feature_registry = FeatureRegistry(mock_sonic_db) + feature_registry.register('test', manifest) + mock_feature_table.set.assert_called_with('test', [ + ('state', 'disabled'), + ('auto_restart', 'enabled'), + ('high_mem_alert', 'disabled'), + ('has_per_asic_scope', 'False'), + ('has_global_scope', 'True'), + ('has_timer', 'True') + ]) diff --git a/src/sonic-package-manager/tests/test_utils.py b/src/sonic-package-manager/tests/test_utils.py new file mode 100644 index 00000000000..c4d8b158408 --- /dev/null +++ b/src/sonic-package-manager/tests/test_utils.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +from sonic_package_manager import utils + + +def test_make_python_identifier(): + assert utils.make_python_identifier('-some-package name').isidentifier() + assert utils.make_python_identifier('01 leading digit').isidentifier()