SONiC Management Framework Release 1.0

.gitmodules

@@ -66,6 +66,12 @@
 [submodule "platform/mellanox/mlnx-sai/SAI-Implementation"]
 	path = platform/mellanox/mlnx-sai/SAI-Implementation
 	url = https://github.com/Mellanox/SAI-Implementation
+[submodule "src/sonic-mgmt-framework"]
+	path = src/sonic-mgmt-framework
+	url = https://github.com/Azure/sonic-mgmt-framework
+[submodule "src/sonic-telemetry"]
+	path = src/sonic-telemetry
+	url = https://github.com/Azure/sonic-telemetry
 [submodule "Switch-SDK-drivers"]
 	path = platform/mellanox/sdk-src/sx-kernel/Switch-SDK-drivers
 	url = https://github.com/Mellanox/Switch-SDK-drivers

dockers/docker-sonic-mgmt-framework/Dockerfile.j2

@@ -0,0 +1,42 @@
+FROM docker-config-engine-stretch
+
+ARG docker_container_name
+RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update
+RUN pip install connexion==1.1.15 \
+		setuptools==21.0.0 \
+		grpcio-tools==1.20.0 \
+		pyangbind==0.6.0 \
+		certifi==2017.4.17 \
+		python-dateutil==2.6.0 \
+		six==1.11.0 \
+		urllib3==1.21.1
+
+
+
+## Install redis-tools dependencies
+## TODO: implicitly install dependencies
+RUN apt-get -y install libjemalloc1 libatomic1 liblua5.1-0 lua-bitop lua-cjson
+
+COPY \
+{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%}
+debs/{{ deb }}{{' '}}
+{%- endfor -%}
+debs/
+
+RUN dpkg -i \
+{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%}
+debs/{{ deb }}{{' '}}
+{%- endfor %}
+
+COPY ["start.sh", "rest-server.sh", "/usr/bin/"]
+COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
+
+RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
+RUN rm -rf /debs
+
+ENTRYPOINT ["/usr/bin/supervisord"]

dockers/docker-sonic-mgmt-framework/base_image_files/sonic-cli

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+docker exec -it mgmt-framework /usr/sbin/cli/clish_start "$@"
+

dockers/docker-sonic-mgmt-framework/rest-server.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+# Startup script for SONiC Management REST Server
+
+SERVER_PORT=
+LOG_LEVEL=
+CLIENT_AUTH=
+SERVER_CRT=
+SERVER_KEY=
+CA_CERT=
+
+# Read basic server settings from REST_SERVER|default entry
+HAS_REST_CONFIG=$(sonic-cfggen -d -v "1 if REST_SERVER and REST_SERVER['default']")
+if [ "$HAS_REST_CONFIG" == "1" ]; then
+    SERVER_PORT=$(sonic-cfggen -d -v "REST_SERVER['default']['port']")
+    CLIENT_AUTH=$(sonic-cfggen -d -v "REST_SERVER['default']['client_auth']")
+    LOG_LEVEL=$(sonic-cfggen -d -v "REST_SERVER['default']['log_level']")
+fi
+
+# Read certificate file paths from DEVICE_METADATA|x509 entry.
+HAS_X509_CONFIG=$(sonic-cfggen -d -v "1 if DEVICE_METADATA and DEVICE_METADATA['x509']")
+if [ "$HAS_X509_CONFIG" == "1" ]; then
+    SERVER_CRT=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_crt']")
+    SERVER_KEY=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_key']")
+    CA_CRT=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['ca_crt']")
+fi
+
+# Create temporary server certificate if they not configured in ConfigDB
+if [ -z $SERVER_CRT ] && [ -z $SERVER_KEY ]; then
+    echo "Generating temporary TLS server certificate ..."
+    (cd /tmp && /usr/sbin/generate_cert --host="localhost,127.0.0.1")
+    SERVER_CRT=/tmp/cert.pem
+    SERVER_KEY=/tmp/key.pem
+fi
+
+
+REST_SERVER_ARGS="-ui /rest_ui -logtostderr"
+[ ! -z $SERVER_PORT ] && REST_SERVER_ARGS+=" -port $SERVER_PORT"
+[ ! -z $LOG_LEVEL   ] && REST_SERVER_ARGS+=" -v $LOG_LEVEL"
+[ ! -z $CLIENT_AUTH ] && REST_SERVER_ARGS+=" -client_auth $CLIENT_AUTH"
+[ ! -z $SERVER_CRT  ] && REST_SERVER_ARGS+=" -cert $SERVER_CRT"
+[ ! -z $SERVER_KEY  ] && REST_SERVER_ARGS+=" -key $SERVER_KEY"
+[ ! -z $CA_CRT      ] && REST_SERVER_ARGS+=" -cacert $CA_CRT"
+
+echo "REST_SERVER_ARGS = $REST_SERVER_ARGS"
+
+
+export CVL_SCHEMA_PATH=/usr/sbin/schema
+export LIBYANG_EXTENSIONS_PLUGINS_DIR=/usr/lib/x86_64-linux-gnu/libyang/extensions
+export LIBYANG_USER_TYPES_PLUGINS_DIR=/usr/lib/x86_64-linux-gnu/libyang/user_types
+
+exec /usr/sbin/rest_server ${REST_SERVER_ARGS}

dockers/docker-sonic-mgmt-framework/start.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+mkdir -p /var/sonic
+echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status
+
+rm -f /var/run/rsyslogd.pid
+
+supervisorctl start rsyslogd
+
+supervisorctl start rest-server

dockers/docker-sonic-mgmt-framework/supervisord.conf

@@ -0,0 +1,28 @@
+[supervisord]
+logfile_maxbytes=1MB
+logfile_backups=2
+nodaemon=true
+
+[program:start.sh]
+command=/usr/bin/start.sh
+priority=1
+autostart=true
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n
+priority=2
+autostart=false
+autorestart=false
+stdout_logfile=syslog
+stderr_logfile=syslog
+
+[program:rest-server]
+command=/usr/bin/rest-server.sh
+priority=3
+autostart=false
+autorestart=true
+stdout_logfile=syslog
+stderr_logfile=syslog

dockers/docker-sonic-telemetry/dialout.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
 # Start with default config
-
+export CVL_SCHEMA_PATH=/usr/sbin/schema
 exec /usr/sbin/dialout_client_cli -insecure -logtostderr  -v 2
 

dockers/docker-sonic-telemetry/supervisord.conf

@@ -6,8 +6,8 @@ nodaemon=true
 [eventlistener:supervisor-proc-exit-listener]
 command=/usr/bin/supervisor-proc-exit-listener
 events=PROCESS_STATE_EXITED
-autostart=always
-autorestart=unexpected
+autostart=true
+autorestart=false
 
 [program:start.sh]
 command=/usr/bin/start.sh

dockers/docker-sonic-telemetry/telemetry.sh

@@ -6,6 +6,7 @@ X509=`sonic-cfggen -d -v "DEVICE_METADATA['x509']"`
 TELEMETRY=`sonic-cfggen -d -v 'TELEMETRY.keys() | join(" ") if TELEMETRY'`
 
 TELEMETRY_ARGS=" -logtostderr"
+export CVL_SCHEMA_PATH=/usr/sbin/schema
 
 if [ -n "$X509" ]; then
 	SERVER_CRT=`sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_crt']"`

files/build_templates/mgmt-framework.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Management Framework container
+Requires=swss.service
+After=swss.service
+Before=ntp-config.service
+
+[Service]
+User={{ sonicadmin_user }}
+ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
+ExecStart=/usr/bin/{{docker_container_name}}.sh wait
+ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+
+[Install]
+WantedBy=multi-user.target

files/image_config/platform/rc.local

@@ -331,4 +331,7 @@ if [ -f $FIRST_BOOT_FILE ]; then
     firsttime_exit
 fi
 
+# Create dir where following scripts put their output files
+mkdir -p /var/platform
+
 exit 0

rules/config

@@ -98,3 +98,6 @@ DEFAULT_VS_PREPARE_MEM = yes
 
 # ENABLE_SYSTEM_SFLOW - build docker-sonic-sflow for sFlow support
 ENABLE_SFLOW = y
+
+# ENABLE_MGMT_FRAMEWORK - build docker-sonic-mgt-framework for CLI and REST server support
+ENABLE_MGMT_FRAMEWORK = y

rules/docker-sonic-mgmt-framework.mk

@@ -0,0 +1,34 @@
+# docker image for mgmt-framework
+
+DOCKER_MGMT_FRAMEWORK_STEM = docker-sonic-mgmt-framework
+DOCKER_MGMT_FRAMEWORK = $(DOCKER_MGMT_FRAMEWORK_STEM).gz
+DOCKER_MGMT_FRAMEWORK_DBG = $(DOCKER_MGMT_FRAMEWORK_STEM)-$(DBG_IMAGE_MARK).gz
+
+$(DOCKER_MGMT_FRAMEWORK)_PATH = $(DOCKERS_PATH)/$(DOCKER_MGMT_FRAMEWORK_STEM)
+
+$(DOCKER_MGMT_FRAMEWORK)_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT_FRAMEWORK)
+$(DOCKER_MGMT_FRAMEWORK)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS)
+$(DOCKER_MGMT_FRAMEWORK)_DBG_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT_FRAMEWORK_DBG)
+
+SONIC_DOCKER_IMAGES += $(DOCKER_MGMT_FRAMEWORK)
+$(DOCKER_MGMT_FRAMEWORK)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH)
+$(DOCKER_MGMT_FRAMEWORK)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES)
+
+ifeq ($(ENABLE_MGMT_FRAMEWORK), y)
+SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_MGMT_FRAMEWORK)
+SONIC_STRETCH_DOCKERS += $(DOCKER_MGMT_FRAMEWORK)
+endif
+
+SONIC_DOCKER_DBG_IMAGES += $(DOCKER_MGMT_FRAMEWORK_DBG)
+ifeq ($(ENABLE_MGMT_FRAMEWORK), y)
+SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_MGMT_FRAMEWORK_DBG)
+SONIC_STRETCH_DBG_DOCKERS += $(DOCKER_MGMT_FRAMEWORK_DBG)
+endif
+
+$(DOCKER_MGMT_FRAMEWORK)_CONTAINER_NAME = mgmt-framework
+$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += --net=host --privileged -t
+$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /etc:/host_etc:ro
+$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/"
+
+$(DOCKER_MGMT_FRAMEWORK)_BASE_IMAGE_FILES += sonic-cli:/usr/bin/sonic-cli

rules/docker-telemetry.mk

@@ -27,5 +27,6 @@ endif
 $(DOCKER_TELEMETRY)_CONTAINER_NAME = telemetry
 $(DOCKER_TELEMETRY)_RUN_OPT += --net=host --privileged -t
 $(DOCKER_TELEMETRY)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_TELEMETRY)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/"
 
 $(DOCKER_TELEMETRY)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)

rules/sonic-mgmt-framework.mk

@@ -0,0 +1,16 @@
+# SONiC mgmt-framework package
+
+ifeq ($(ENABLE_MGMT_FRAMEWORK), y)
+
+SONIC_MGMT_FRAMEWORK = sonic-mgmt-framework_1.0-01_amd64.deb
+$(SONIC_MGMT_FRAMEWORK)_SRC_PATH = $(SRC_PATH)/sonic-mgmt-framework
+$(SONIC_MGMT_FRAMEWORK)_DEPENDS = $(LIBYANG_DEV) $(LIBYANG)
+$(SONIC_MGMT_FRAMEWORK)_RDEPENDS = $(LIBYANG)
+SONIC_DPKG_DEBS += $(SONIC_MGMT_FRAMEWORK)
+
+SONIC_MGMT_FRAMEWORK_DBG = sonic-mgmt-framework-dbg_1.0-01_amd64.deb
+$(SONIC_MGMT_FRAMEWORK_DBG)_DEPENDS += $(SONIC_MGMT_FRAMEWORK)
+$(SONIC_MGMT_FRAMEWORK_DBG)_RDEPENDS += $(SONIC_MGMT_FRAMEWORK)
+$(eval $(call add_derived_package,$(SONIC_MGMT_FRAMEWORK),$(SONIC_MGMT_FRAMEWORK_DBG)))
+
+endif

rules/telemetry.mk

@@ -1,5 +1,7 @@
 # SONiC telemetry package
 
 SONIC_TELEMETRY = sonic-telemetry_0.1_$(CONFIGURED_ARCH).deb
-$(SONIC_TELEMETRY)_SRC_PATH = $(SRC_PATH)/telemetry
+$(SONIC_TELEMETRY)_SRC_PATH = $(SRC_PATH)/sonic-telemetry
+$(SONIC_TELEMETRY)_DEPENDS = $(LIBYANG_DEV) $(LIBYANG)
+$(SONIC_TELEMETRY)_RDEPENDS = $(LIBYANG)
 SONIC_DPKG_DEBS += $(SONIC_TELEMETRY)

sonic-slave-stretch/Dockerfile.j2

@@ -352,6 +352,9 @@ RUN pip install mockredispy==2.9.3
 RUN pip install pytest-runner==4.4
 RUN pip install setuptools==40.8.0
 
+# For mgmt-framework build
+RUN pip install mmh3
+
 # Install dependencies for isc-dhcp-relay build
 RUN apt-get -y build-dep isc-dhcp