Sync to master

.gitignore

@@ -1,4 +1,8 @@
+# OS-generated files
+.DS_Store
+
 # Build system related
+.arch
 .platform
 .screen
 
@@ -9,6 +13,12 @@ target/
 *.deb
 *.changes
 *.buildinfo
+*.tar
+*.xz
+*.gz
+*-dbg
+*dbg.j2
+*.img
 
 # Subdirectories in src
 src/bash/*
@@ -43,6 +53,8 @@ src/libnl3/*
 !src/libnl3/Makefile
 src/libteam/*
 !src/libteam/Makefile
+src/libyang/*
+!src/libyang/Makefile
 src/lldpd/*
 !src/lldpd/Makefile
 !src/lldpd/patch/
@@ -59,13 +71,23 @@ src/radvd/*
 !src/radvd/patch/
 src/redis/*
 !src/redis/Makefile
+src/smartmontools/*
+!src/smartmontools/Makefile
 src/snmpd/*
 !src/snmpd/Makefile
 src/sonic-device-data/src/device/
 src/sonic-device-data/src/debian/
 src/supervisor/*
 !src/supervisor/Makefile
 !src/supervisor/patch/
+src/swig/*
+!src/swig/Makefile
+src/systemd-sonic-generator/systemd-sonic-generator
+src/systemd-sonic-generator/debian/*
+!src/systemd-sonic-generator/debian/changelog
+!src/systemd-sonic-generator/debian/compat
+!src/systemd-sonic-generator/debian/control
+!src/systemd-sonic-generator/debian/rules
 src/telemetry/debian/*
 !src/telemetry/debian/changelog
 !src/telemetry/debian/compat
@@ -77,6 +99,8 @@ src/thrift/*
 !src/thrift/Makefile
 
 # Autogenerated Dockerfiles
+sonic-slave/Dockerfile
+sonic-slave-stretch/Dockerfile
 dockers/docker-base/Dockerfile
 dockers/docker-base-stretch/Dockerfile
 dockers/docker-config-engine/Dockerfile
@@ -109,3 +133,7 @@ src/sonic-config-engine/sonic_config_engine.egg-info
 src/sonic-daemon-base/**/*.pyc
 src/sonic-daemon-base/build
 src/sonic-daemon-base/sonic_daemon_base.egg-info
+
+# Misc. files
+files/initramfs-tools/arista-convertfs
+files/initramfs-tools/union-mount

.gitmodules

@@ -47,6 +47,7 @@
 [submodule "src/sonic-frr/frr"]
 	path = src/sonic-frr/frr
 	url = https://github.com/Azure/sonic-frr.git
+	branch = frr/7.0
 [submodule "platform/p4/p4-hlir/p4-hlir-v1.1"]
 	path = platform/p4/p4-hlir/p4-hlir-v1.1
 	url = https://github.com/p4lang/p4-hlir.git

Makefile

@@ -1,20 +1,20 @@
 # SONiC make file
 
-NOSTRETCH ?= 0
+NOJESSIE ?= 0
 
 %::
 	@echo "+++ --- Making $@ --- +++"
-ifeq ($(NOSTRETCH), 0)
-	BLDENV=stretch make -f Makefile.work stretch
+ifeq ($(NOJESSIE), 0)
+	EXTRA_JESSIE_TARGETS=$(notdir $@) make -f Makefile.work jessie
 endif
-	make -f Makefile.work $@
+	BLDENV=stretch make -f Makefile.work $@
 
-stretch:
+jessie:
 	@echo "+++ Making $@ +++"
-ifeq ($(NOSTRETCH), 0)
-	BLDENV=stretch make -f Makefile.work stretch
+ifeq ($(NOJESSIE), 0)
+	make -f Makefile.work jessie
 endif
 
 clean reset init configure showtag sonic-slave-build sonic-slave-bash :
 	@echo "+++ Making $@ +++"
-	make -f Makefile.work $@
+	BLDENV=stretch make -f Makefile.work $@

Makefile.work

@@ -20,11 +20,10 @@
 #  *                Please note that with current Stretch build structure,
 #  *                user of KEEP_SLAVE_ON feature will have to be conscious
 #  *                about which docker to stay inside after build is done.
-#  *                - If user desires to stay inside Stretch docker, please issue
-#  *                  make KEEP_SLAVE_ON=yes stretch
 #  *                - If user desires to stay inside Jessie docker, please issue
-#  *                  (a successful "make stretch" may be needed before the following command)
-#  *                  make NOSTRETCH=1 KEEP_SLAVE_ON=yes <any jessie target>
+#  *                  make KEEP_SLAVE_ON=yes jessie
+#  *                - If user desires to stay inside Stretch docker, please issue
+#  *                  make NOJESSIE=1 KEEP_SLAVE_ON=yes <any target>
 #  * SOURCE_FOLDER: host path to be mount as /var/$(USER)/src, only effective when KEEP_SLAVE_ON=yes
 #  * SONIC_BUILD_JOBS: Specifying number of concurrent build job(s) to run
 #  * VS_PREPARE_MEM: Prepare memory in VS build (drop cache and compact).
@@ -43,26 +42,50 @@ ifeq ($(USER), root)
 $(error Add your user account to docker group and use your user account to make. root or sudo are not supported!)
 endif
 
+# Check for j2cli availability
+J2_VER := $(shell j2 --version 2>&1 | grep j2cli | awk '{printf $$2}')
+ifeq ($(J2_VER),)
+$(error Install j2cli)
+endif
+
+# Check for minimum Docker version on build host
+# Note: Using the greater of CE (17.05.0) and EE (17.06.1) versions that support ARG before FROM
+docker_min := 17.06.1
+docker_min_ver := $(shell echo "$(docker_min)" | awk -F. '{printf("%d%03d%03d\n",$$1,$$2,$$3);}' 2>/dev/null)
+docker_ver := $(shell docker info 2>/dev/null | grep -i "server version" | rev | cut -d' ' -f1 | rev | awk -F. '{printf("%d%03d%03d\n",$$1,$$2,$$3);}' 2>/dev/null)
+docker_is_valid := $(shell if [[ "$(docker_ver)" -lt $(docker_min_ver) ]] ; then echo "0"; else echo "1"; fi)
+ifeq (0,$(docker_is_valid))
+$(error SONiC requires Docker version $(docker_min) or later)
+endif
+
 # Remove lock file in case previous run was forcefully stopped
 $(shell rm -f .screen)
 
 MAKEFLAGS += -B
 
+CONFIGURED_ARCH := $(shell [ -f .arch ] && cat .arch || echo $(PLATFORM_ARCH))
+ifeq ($(CONFIGURED_ARCH),)
+    override CONFIGURED_ARCH = amd64
+endif
+ifeq ($(PLATFORM_ARCH),)
+	override PLATFORM_ARCH = $(CONFIGURED_ARCH)
+endif
+
 ifeq ($(BLDENV), stretch)
-SLAVE_BASE_TAG = $(shell sha1sum sonic-slave-stretch/Dockerfile | awk '{print substr($$1,0,11);}')
-SLAVE_TAG = $(shell cat sonic-slave-stretch/Dockerfile.user sonic-slave-stretch/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}')
-SLAVE_BASE_IMAGE = sonic-slave-stretch-base
-SLAVE_IMAGE = sonic-slave-stretch-$(USER)
 SLAVE_DIR = sonic-slave-stretch
 else
-SLAVE_BASE_TAG = $(shell sha1sum sonic-slave/Dockerfile | awk '{print substr($$1,0,11);}')
-SLAVE_TAG = $(shell cat sonic-slave/Dockerfile.user sonic-slave/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}')
-SLAVE_BASE_IMAGE = sonic-slave-base
-SLAVE_IMAGE = sonic-slave-$(USER)
 SLAVE_DIR = sonic-slave
 endif
+SLAVE_BASE_TAG = $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile && sha1sum $(SLAVE_DIR)/Dockerfile | awk '{print substr($$1,0,11);}')
+SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}')
+SLAVE_BASE_IMAGE = $(SLAVE_DIR)-base
+SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER)
 
-OVERLAY_MODULE_CHECK := lsmod | grep "^overlay " > /dev/null 2>&1 || (echo "ERROR: Module 'overlay' not loaded. Try running 'sudo modprobe overlay'."; exit 1)
+OVERLAY_MODULE_CHECK := \
+    lsmod | grep -q "^overlay " &>/dev/null || \
+    zgrep -q 'CONFIG_OVERLAY_FS=y' /proc/config.gz &>/dev/null || \
+    grep -q 'CONFIG_OVERLAY_FS=y' /boot/config-$(shell uname -r) &>/dev/null || \
+    (echo "ERROR: Module 'overlay' not loaded. Try running 'sudo modprobe overlay'."; exit 1)
 
 BUILD_TIMESTAMP := $(shell date +%Y%m%d\.%H%M%S)
 
@@ -86,28 +109,55 @@ include rules/config
 ifeq ($(SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD), y)
     DOCKER_RUN += -v /var/run/docker.sock:/var/run/docker.sock
 endif
+ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64))
+    # Multiarch docker cannot start dockerd service due to iptables cannot run over different arch kernel
+    SONIC_SERVICE_DOCKERD_FOR_MULTIARCH=y
+    SONIC_NATIVE_DOCKERD_FOR_MUTLIARCH := dockerd  --experimental=true --storage-driver=vfs \
+	--data-root=/var/lib/march/docker/ --exec-root=/var/run/march/docker/ \
+	-H unix:///var/run/march/docker.sock -p /var/run/march/docker.pid
+    DOCKER_RUN += -v /var/run/march/docker.sock:/var/run/docker.sock
+    DOCKER_RUN += -v /var/run/march/docker.pid:/var/run/docker.pid
+    DOCKER_RUN += -v /var/run/march/docker:/var/run/docker
+    DOCKER_RUN += -v /var/lib/march/docker:/var/lib/docker
+    SONIC_USERFACL_DOCKERD_FOR_MUTLIARCH := setfacl -m user:$(USER):rw /var/run/march/docker.sock
+
+    #Override Native config to prevent docker service
+    SONIC_CONFIG_USE_NATIVE_DOCKERD_FOR_BUILD=y
+
+    DOCKER_MULTIARCH_CHECK := docker inspect --type image multiarch/qemu-user-static:register &> /dev/null || (echo "multiarch docker not found ..."; docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes)
+
+    DOCKER_SERVICE_MULTIARCH_CHECK := docker -H unix:///var/run/march/docker.sock info &> /dev/null || (echo "Docker march service not running..."; sudo rm -fr /var/run/march/*; (sudo $(SONIC_NATIVE_DOCKERD_FOR_MUTLIARCH) &) &>/dev/null ; sleep 1; sudo $(SONIC_USERFACL_DOCKERD_FOR_MUTLIARCH);)
+
+   # Docker service to load the compiled dockers-*.gz 
+    SONIC_NATIVE_DOCKERD_FOR_DOCKERFS := rm -fr $(PWD)/dockerfs/; mkdir -p $(PWD)/dockerfs/; sudo dockerd --storage-driver=overlay2 --iptables=false \
+	--data-root $(PWD)/dockerfs/var/lib/docker/ --exec-root=$(PWD)/dockerfs/var/run/docker/ \
+	-H unix://$(PWD)/dockerfs/var/run/docker.sock -p $(PWD)/dockerfs/var/run/docker.pid &
+    SONIC_USERFACL_DOCKERD_FOR_DOCKERFS := setfacl -m user:$(USER):rw $(PWD)/dockerfs/var/run/docker.sock
+    DOCKER_SERVICE_DOCKERFS_CHECK := (sudo docker -H unix://$(PWD)/dockerfs/var/run/docker.sock info &> /dev/null && sudo kill -9 `sudo cat $(PWD)/dockerfs/var/run/docker.pid`  && false) || (echo "Starting docker build service..."; (sudo $(SONIC_NATIVE_DOCKERD_FOR_DOCKERFS) ) &>/dev/null ; sleep 1; sudo $(SONIC_USERFACL_DOCKERD_FOR_DOCKERFS);)
+
+endif
 
 DOCKER_BASE_BUILD = docker build --no-cache \
-		    -t $(SLAVE_BASE_IMAGE) \
+		    -t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
 		    --build-arg http_proxy=$(http_proxy) \
 		    --build-arg https_proxy=$(https_proxy) \
-		    $(SLAVE_DIR) && \
-		    docker tag $(SLAVE_BASE_IMAGE):latest $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG)
+		    $(SLAVE_DIR)
 
 DOCKER_BUILD = docker build --no-cache \
 	       --build-arg user=$(USER) \
 	       --build-arg uid=$(shell id -u) \
 	       --build-arg guid=$(shell id -g) \
 	       --build-arg hostname=$(shell echo $$HOSTNAME) \
-	       -t $(SLAVE_IMAGE) \
+               --build-arg slave_base_tag_ref=$(SLAVE_BASE_TAG) \
+	       -t $(SLAVE_IMAGE):$(SLAVE_TAG) \
 	       -f $(SLAVE_DIR)/Dockerfile.user \
-	       $(SLAVE_DIR) && \
-	       docker tag $(SLAVE_IMAGE):latest $(SLAVE_IMAGE):$(SLAVE_TAG)
+	       $(SLAVE_DIR)
 
 SONIC_BUILD_INSTRUCTION :=  make \
                            -f slave.mk \
                            BLDENV=$(BLDENV) \
                            PLATFORM=$(PLATFORM) \
+                           PLATFORM_ARCH=$(PLATFORM_ARCH) \
                            BUILD_NUMBER=$(BUILD_NUMBER) \
                            BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) \
                            ENABLE_DHCP_GRAPH_SERVICE=$(ENABLE_DHCP_GRAPH_SERVICE) \
@@ -118,19 +168,27 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            PASSWORD=$(PASSWORD) \
                            USERNAME=$(USERNAME) \
                            SONIC_BUILD_JOBS=$(SONIC_BUILD_JOBS) \
+                           SONIC_USE_DOCKER_BUILDKIT=$(SONIC_USE_DOCKER_BUILDKIT) \
                            VS_PREPARE_MEM=$(VS_PREPARE_MEM) \
                            KERNEL_PROCURE_METHOD=$(KERNEL_PROCURE_METHOD) \
                            HTTP_PROXY=$(http_proxy) \
                            HTTPS_PROXY=$(https_proxy) \
                            SONIC_ENABLE_SYSTEM_TELEMETRY=$(ENABLE_SYSTEM_TELEMETRY) \
+                           EXTRA_JESSIE_TARGETS=$(EXTRA_JESSIE_TARGETS) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset
 
 .DEFAULT_GOAL :=  all
 
 %::
+ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64))
+	@$(DOCKER_MULTIARCH_CHECK)
+	@$(DOCKER_SERVICE_MULTIARCH_CHECK)
+	@$(DOCKER_SERVICE_DOCKERFS_CHECK)
+endif
 	@$(OVERLAY_MODULE_CHECK)
+
 	@docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \
 	    { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \
 	    $(DOCKER_BASE_BUILD) ; }
@@ -169,18 +227,24 @@ init :
 	@git submodule update --init --recursive
 	@git submodule foreach --recursive '[ -f .git ] && echo "gitdir: $$(realpath --relative-to=. $$(cut -d" " -f2 .git))" > .git'
 
+.ONESHELL : reset
 reset :
 	@echo && echo -n "Warning! All local changes will be lost. Proceed? [y/N]: "
-	@read ans && \
-	    if [ $$ans == y ]; then \
-	        echo "Resetting local repository. Please wait..."; \
-	        $(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) sudo rm -rf fsroot; \
-	        git clean -xfdf; \
-	        git reset --hard; \
-	        git submodule foreach --recursive git clean -xfdf; \
-	        git submodule foreach --recursive git reset --hard; \
-	        git submodule update --init --recursive; \
-			echo "Reset complete!"; \
-	    else \
-	        echo "Reset aborted"; \
-	    fi
+	@read ans && (
+	    if [ $$ans == y ]; then
+	        echo "Resetting local repository. Please wait...";
+	        $(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) sudo rm -rf fsroot;
+	        if [[ "$(CONFIGURED_ARCH)" == "armhf" || "$(CONFIGURED_ARCH)" ==  "arm64" ]]; then
+	            echo "Stopping march $(CONFIGURED_ARCH) docker"
+	            sudo kill -9 `sudo cat /var/run/march/docker.pid` || true
+	            sudo rm -f /var/run/march/docker.pid || true
+	        fi
+	        git clean -xfdf;
+	        git reset --hard;
+	        git submodule foreach --recursive git clean -xfdf;
+	        git submodule foreach --recursive git reset --hard;
+	        git submodule update --init --recursive;
+	        echo "Reset complete!";
+	    else
+	        echo "Reset aborted";
+	    fi )

README.buildsystem.md

@@ -207,3 +207,74 @@ $ make list
 ```
 
 All target groups are used by one or another recipe, so use those recipes as a reference when adding new ones.
+
+## Build debug dockers and debug SONiC installer image:
+Using 'INSTALL_DEBUG_TOOLS=y' builds the debug image.
+
+For example:
+`INSTALL_DEBUG_TOOLS=y make target/sonic-broadcom.bin` 
+
+* Builds debug docker images.
+* Debug images carry a suffix of "-dbg"
+    * e.g. target/docker-orchagent-dbg.gz
+* target/sonic-broadcom.bin is built using debug docker images
+    * Selective sources are archived and available under /src
+    * An empty /debug dir is created for use during debug session.
+    * All debug dockers are mounted with /src:ro and /debug:rw
+    * Login banner will briefly describe these features.
+
+
+_Note: target/sonic-broadcom.bin name is the same irrespective of built using debug or non-debug dockers._
+
+_Recommend: Rename image built using INSTALL_DEBUG_TOOLS=y to mark it explicit. May be `mv target/sonic-broadcom.bin target/sonic-broadcom-dbg.bin`_
+
+### Debug dockers
+* Built with all available debug symbols.
+* Installed with many basic packages that would be required for debugging
+    * gdb
+    * gdbserver
+    * vim
+    * strace
+    * openssh-client
+    * sshpass
+* Loadable into any environment that supports docker
+* Outside SONiC image, you may run the docker with `--entrypoint=/bin/bash` 
+* Use -v to map any of your host directories
+* To debug a core file in non-SONiC environment that supports docker
+    * `docker load -i docker-<name>-dbg.gz`
+    * copy your unzipped core file into ~/debug
+    * `docker run -it -entrypoint=/bin/bash -v ~/debug:/debug <image id>`
+    * `gdb /usr/bin/<your binary> -c /debug/<your core>`
+
+### Debug SONiC image
+
+* Install this image into the switch that supports this image.
+	* For platform independent binary, you may use a debug image for virtual switch
+* Open the archive in /src, if you would need source code for debugging
+* Every debug enabled docker is mounted with /src as read-only
+* The host has /debug dir and it is mapped into every debuggable docker as /debug with read-write permission.
+* To debug a core
+    * Copy core into /debug of host and unzip it
+        * Feel free to create & use sub-dirs as needed.
+        * Entire /debug is mounted inside docker
+    * Get into the docker (`docker -it exec <name> bash`)
+    * `gdb /usr/bin/<binary> -c /debug/<core file path>`
+    * Use set-directory in gdb to map appropriate source dir from under /src
+    * You may set gdb logs to go into /debug
+* For live debugging
+    * Use this as a regular switch
+    * Get into docker, and you may
+        * start process under dbg
+        * attach gdb to running process
+    * Set required source dir from under /src as needed
+    * May use /debug to record all geb logs or any spew from debug session.
+
+  ### To enhance debug dockers
+  * Add to `<docker name>_DBG_IMAGE_PACKAGES`, additional debug tools that will be pre-installed during build.
+	* e.g. `$(DOCKER_ORCHAGENT)_DBG_IMAGE_PACKAGES += perl`
+	* Build will install these tools using "apt"
+  * Add to `<docker name>_DBG_DEPENDS`, additional debug .deb packages that will be pre-installed during build.
+	* e.g. `$(DOCKER_ORCHAGENT)_DBG_DEPENDS +=  $(SWSS_DBG) $(LIBSWSSCOMMON_DBG) $(LIBSAIREDIS_DBG)`
+  * Add to `DBG_SRC_ARCHIVE`, the source dirs to archive
+	* The source files (.c, .cpp, .h & .hpp) under this dir tree are archived.
+	* e.g. rules/swss.mk has `DBG_SRC_ARCHIVE += swss`