From 0c07547dc622a51ddf527fab73071c3e32d1826a Mon Sep 17 00:00:00 2001 From: lguohan Date: Wed, 30 Dec 2020 04:58:20 -0800 Subject: [PATCH 1/5] [build]: change user name to lower case when used in sonic-slave tag (#6319) sonic-slave tag only allows all lower case. In case the user name is mixed case, we need to change user name to all lower case. Signed-off-by: Guohan Lu --- Makefile.work | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile.work b/Makefile.work index 1734c68a49d..323d05f027a 100644 --- a/Makefile.work +++ b/Makefile.work @@ -39,6 +39,7 @@ SHELL = /bin/bash USER := $(shell id -un) PWD := $(shell pwd) +USER_LC := $(shell echo $(USER) | tr A-Z a-z) ifeq ($(USER), root) $(error Add your user account to docker group and use your user account to make. root or sudo are not supported!) @@ -81,7 +82,7 @@ endif SLAVE_BASE_TAG = $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile && sha1sum $(SLAVE_DIR)/Dockerfile | awk '{print substr($$1,0,11);}') SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}') SLAVE_BASE_IMAGE = $(SLAVE_DIR) -SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER) +SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER_LC) OVERLAY_MODULE_CHECK := \ lsmod | grep -q "^overlay " &>/dev/null || \ @@ -131,7 +132,7 @@ ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64)) DOCKER_SERVICE_SAFE_KILLER := (MARCH_PID=`ps -eo pid,cmd | grep "[0-9] dockerd.*march" | awk '{print $$1}'`; echo "Killing march docker $$MARCH_PID"; [ -z "$$MARCH_PID" ] || sudo kill -9 "$$MARCH_PID";) DOCKER_SERVICE_MULTIARCH_CHECK := ($(DOCKER_SERVICE_SAFE_KILLER); sudo rm -fr /var/run/march/; (echo "Starting docker march service..."; sudo $(SONIC_NATIVE_DOCKERD_FOR_MUTLIARCH) &) &>/dev/null ; sleep 2; sudo $(SONIC_USERFACL_DOCKERD_FOR_MUTLIARCH);) - # Docker service to load the compiled dockers-*.gz + # Docker service to load the compiled dockers-*.gz SONIC_NATIVE_DOCKERD_FOR_DOCKERFS := rm -fr $(PWD)/dockerfs/; mkdir -p $(PWD)/dockerfs/; sudo dockerd --storage-driver=overlay2 --iptables=false \ --data-root $(PWD)/dockerfs/var/lib/docker/ --exec-root=$(PWD)/dockerfs/var/run/docker/ \ -H unix://$(PWD)/dockerfs/var/run/docker.sock -p $(PWD)/dockerfs/var/run/docker.pid & @@ -199,7 +200,7 @@ ifneq ($(BLDENV), ) endif endif @$(OVERLAY_MODULE_CHECK) - + @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \ $(DOCKER_BASE_BUILD) ; } From ec2d805e1b6506be3b7f6e65727d3e45716cb71e Mon Sep 17 00:00:00 2001 From: lguohan Date: Tue, 12 Jan 2021 06:03:12 -0800 Subject: [PATCH 2/5] [build]: fix dpkg admindir corruption issue in parallel build (#6408) Fix #119 when parallel build is enable, multiple dpkg-buildpackage instances are running at the same time. /var/lib/dpkg is shared by all instances and the /var/lib/dpkg/updates could be corrupted and cause the build failure. the fix is to use overlay fs to mount separate /var/lib/dpkg for each dpkg-buildpackage instance so that they are not affecting each other. Signed-off-by: Guohan Lu --- rules/functions | 12 ++++++++++++ slave.mk | 8 ++++++-- src/bash/Makefile | 2 +- src/hiredis/Makefile | 2 +- src/ifupdown2/Makefile | 2 +- src/initramfs-tools/Makefile | 2 +- src/iproute2/Makefile | 2 +- src/iptables/Makefile | 2 +- src/isc-dhcp/Makefile | 2 +- src/libnl3/Makefile | 2 +- src/libteam/Makefile | 2 +- src/lldpd/Makefile | 2 +- src/lm-sensors/Makefile | 2 +- src/monit/Makefile | 2 +- src/mpdecimal/Makefile | 2 +- src/python-click/Makefile | 2 +- src/python3/Makefile | 2 +- src/radvd/Makefile | 2 +- src/redis/Makefile | 2 +- src/sflow/hsflowd/Makefile | 2 +- src/sflow/psample/Makefile | 2 +- src/sflow/sflowtool/Makefile | 2 +- src/smartmontools/Makefile | 2 +- src/snmpd/Makefile | 2 +- src/socat/Makefile | 2 +- src/sonic-device-data/Makefile | 2 +- src/sonic-frr/Makefile | 2 +- src/swig/Makefile | 2 +- src/systemd-sonic-generator/Makefile | 2 +- src/tacacs/nss/Makefile | 2 +- src/tacacs/pam/Makefile | 4 ++-- src/thrift/Makefile | 2 +- 32 files changed, 49 insertions(+), 33 deletions(-) diff --git a/rules/functions b/rules/functions index 93e2a346274..38f2028e011 100644 --- a/rules/functions +++ b/rules/functions @@ -96,3 +96,15 @@ endef ############################################################################### expand = $(foreach d,$(1),$(call expand,$($(d)_$(2)),$(2))) $(1) + +############################################################################### +## Setup overlay fs for dpkg admin directory /var/lib/dpkg +############################################################################### +define SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR +upperdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH)) +workdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH)) +mergedir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH)) +sudo mount -t overlay overlay -olowerdir=/var/lib/dpkg,upperdir=$$upperdir,workdir=$$workdir $$mergedir +export SONIC_DPKG_ADMINDIR=$$mergedir +trap "sudo umount $$mergedir && rm -rf $$mergedir $$upperdir $$workdir" EXIT +endef diff --git a/slave.mk b/slave.mk index 1f3246fd846..b9b5a4a6d25 100644 --- a/slave.mk +++ b/slave.mk @@ -38,6 +38,7 @@ STRETCH_DEBS_PATH = $(TARGET_PATH)/debs/stretch STRETCH_FILES_PATH = $(TARGET_PATH)/files/stretch DBG_IMAGE_MARK = dbg DBG_SRC_ARCHIVE_FILE = $(TARGET_PATH)/sonic_src.tar.gz +DPKG_ADMINDIR_PATH = /sonic/dpkg CONFIGURED_PLATFORM := $(shell [ -f .platform ] && cat .platform || echo generic) PLATFORM_PATH = platform/$(CONFIGURED_PLATFORM) @@ -68,6 +69,7 @@ configure : @mkdir -p target/files/stretch @mkdir -p target/python-debs @mkdir -p target/python-wheels + @mkdir -p $(DPKG_ADMINDIR_PATH) @echo $(PLATFORM) > .platform @echo $(PLATFORM_ARCH) > .arch @@ -331,6 +333,7 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a rm -f $(addprefix $(DEBS_PATH)/, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS)) # Apply series of patches if exist if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && QUILT_PATCHES=../$(notdir $($*_SRC_PATH)).patch quilt push -a; popd; fi + $(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR) # Build project and take package DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC}" make DEST=$(shell pwd)/$(DEBS_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(DEBS_PATH)/$* $(LOG) # Clean up @@ -354,9 +357,10 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a # Build project pushd $($*_SRC_PATH) $(LOG) [ ! -f ./autogen.sh ] || ./autogen.sh $(LOG) + $(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR) $(if $($*_DPKG_TARGET), - DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) $(LOG), - DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) $(LOG) + DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) --admindir $$mergedir $(LOG), + DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $$mergedir $(LOG) ) popd $(LOG) # Clean up diff --git a/src/bash/Makefile b/src/bash/Makefile index 602dc01ece7..6576ff92e74 100644 --- a/src/bash/Makefile +++ b/src/bash/Makefile @@ -10,7 +10,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dget -u https://launchpad.net/debian/+archive/primary/+sourcefiles/bash/$(BASH_VERSION_FULL)/bash_$(BASH_VERSION_FULL).dsc pushd bash-$(BASH_VERSION_MAJOR) - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DEST)/ diff --git a/src/hiredis/Makefile b/src/hiredis/Makefile index 746056e51de..4935a039e0a 100644 --- a/src/hiredis/Makefile +++ b/src/hiredis/Makefile @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x hiredis_$(HIREDIS_VERSION_FULL).dsc pushd hiredis-$(HIREDIS_VERSION) - fakeroot debian/rules -j$(SONIC_CONFIG_MAKE_JOBS) binary + dpkg-buildpackage -rfakeroot -d -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DERIVED_TARGETS) $(DEST)/ diff --git a/src/ifupdown2/Makefile b/src/ifupdown2/Makefile index 66b4df2ec32..89228bedafa 100644 --- a/src/ifupdown2/Makefile +++ b/src/ifupdown2/Makefile @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : pushd ./ifupdown2-$(IFUPDOWN2_VERSION) # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/initramfs-tools/Makefile b/src/initramfs-tools/Makefile index 0b08e4aa431..b412dee54bd 100644 --- a/src/initramfs-tools/Makefile +++ b/src/initramfs-tools/Makefile @@ -19,7 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # Build the package rm -f debian/*.debhelper.log - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/iproute2/Makefile b/src/iproute2/Makefile index 8748550a1d7..5b354ce8b01 100644 --- a/src/iproute2/Makefile +++ b/src/iproute2/Makefile @@ -17,7 +17,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x iproute2_$(IPROUTE2_VERSION_FULL).dsc pushd iproute2-$(IPROUTE2_VERSION) - dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DEST)/ diff --git a/src/iptables/Makefile b/src/iptables/Makefile index 60154c19ddb..b12be000ac9 100644 --- a/src/iptables/Makefile +++ b/src/iptables/Makefile @@ -38,7 +38,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/isc-dhcp/Makefile b/src/isc-dhcp/Makefile index 45535574ed6..6fcba6c6a2f 100644 --- a/src/isc-dhcp/Makefile +++ b/src/isc-dhcp/Makefile @@ -23,7 +23,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/libnl3/Makefile b/src/libnl3/Makefile index a0e9891c9ef..c45ac85b728 100644 --- a/src/libnl3/Makefile +++ b/src/libnl3/Makefile @@ -21,7 +21,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : git checkout tags/libnl$(subst .,_,$(LIBNL3_VERSION_BASE)) ln -s ../debian debian - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/libteam/Makefile b/src/libteam/Makefile index b1ef1237541..bc85e4b5ea8 100644 --- a/src/libteam/Makefile +++ b/src/libteam/Makefile @@ -31,7 +31,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : mv tmp/debian libteam/ rm -rf tmp pushd ./libteam - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/lldpd/Makefile b/src/lldpd/Makefile index 285d7fec991..8971c8f64f2 100644 --- a/src/lldpd/Makefile +++ b/src/lldpd/Makefile @@ -35,7 +35,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - env "with_netlink_receive_bufsize=1024*1024" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + env "with_netlink_receive_bufsize=1024*1024" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/lm-sensors/Makefile b/src/lm-sensors/Makefile index 50cc3e51bb3..1d03526462b 100644 --- a/src/lm-sensors/Makefile +++ b/src/lm-sensors/Makefile @@ -16,7 +16,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dget -u http://deb.debian.org/debian/pool/main/l/lm-sensors/lm-sensors_$(LM_SENSORS_VERSION_FULL).dsc git apply *.patch pushd lm-sensors-$(LM_SENSORS_VERSION) - DEB_BUILD_OPTIONS=nocheck PROG_EXTRA=sensord dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + DEB_BUILD_OPTIONS=nocheck PROG_EXTRA=sensord dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/monit/Makefile b/src/monit/Makefile index 4ad9edd7914..570f30cf60f 100644 --- a/src/monit/Makefile +++ b/src/monit/Makefile @@ -24,7 +24,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/mpdecimal/Makefile b/src/mpdecimal/Makefile index c4ef5b78b49..b7f82ba702e 100644 --- a/src/mpdecimal/Makefile +++ b/src/mpdecimal/Makefile @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x mpdecimal_$(MPDECIMAL_VERSION_FULL).dsc pushd mpdecimal-$(MPDECIMAL_VERSION) - dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DERIVED_TARGETS) $(DEST)/ diff --git a/src/python-click/Makefile b/src/python-click/Makefile index 4deb27257fd..31716e03616 100644 --- a/src/python-click/Makefile +++ b/src/python-click/Makefile @@ -19,7 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : git reset --hard debian/$(PYTHON_CLICK_VERSION) # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb package to the destination directory diff --git a/src/python3/Makefile b/src/python3/Makefile index c5d8c386c1b..1c37e9d06c4 100644 --- a/src/python3/Makefile +++ b/src/python3/Makefile @@ -35,7 +35,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : fi done - dpkg-buildpackage -rfakeroot -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd cp $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/radvd/Makefile b/src/radvd/Makefile index 98e62bc216d..6d3cfde2b5c 100644 --- a/src/radvd/Makefile +++ b/src/radvd/Makefile @@ -25,7 +25,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg import -s ../patch/series # Build source and Debian packages - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb package to the destination directory diff --git a/src/redis/Makefile b/src/redis/Makefile index e2d9e5828f5..3a4fe3f4844 100644 --- a/src/redis/Makefile +++ b/src/redis/Makefile @@ -22,7 +22,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : pushd redis-$(REDIS_VERSION) export ARCH="" - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/sflow/hsflowd/Makefile b/src/sflow/hsflowd/Makefile index 0a8094e3b69..c20bf854098 100644 --- a/src/sflow/hsflowd/Makefile +++ b/src/sflow/hsflowd/Makefile @@ -21,7 +21,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : chmod u+x debian/rules sed -i -e s/_VERSION_/$(HSFLOWD_VERSION)-$(HSFLOWD_SUBVERSION)/g debian/changelog - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --buildinfo-option=-u. --changes-option=-u. + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --buildinfo-option=-u. --changes-option=-u. --admindir $(SONIC_DPKG_ADMINDIR) mv $(DERIVED_TARGET) $* $(DEST)/ popd diff --git a/src/sflow/psample/Makefile b/src/sflow/psample/Makefile index 3f03c249989..bbdefde2289 100644 --- a/src/sflow/psample/Makefile +++ b/src/sflow/psample/Makefile @@ -13,7 +13,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : pushd ./libpsample git checkout -b libpsample -f e48fad2 - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DEST)/ diff --git a/src/sflow/sflowtool/Makefile b/src/sflow/sflowtool/Makefile index 10a3f2d24de..5f2ad995a58 100644 --- a/src/sflow/sflowtool/Makefile +++ b/src/sflow/sflowtool/Makefile @@ -12,7 +12,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : pushd ./sflowtool git checkout -b sflowtool -f 6c2963b - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DEST)/ diff --git a/src/smartmontools/Makefile b/src/smartmontools/Makefile index 4ac3d6ac315..8f0f0695659 100644 --- a/src/smartmontools/Makefile +++ b/src/smartmontools/Makefile @@ -13,7 +13,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x smartmontools_$(SMARTMONTOOLS_VERSION_FULL).dsc pushd smartmontools-$(SMARTMONTOOLS_VERSION_MAJOR) - dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $* $(DEST)/ diff --git a/src/snmpd/Makefile b/src/snmpd/Makefile index 77084594d8b..a09735bb8af 100644 --- a/src/snmpd/Makefile +++ b/src/snmpd/Makefile @@ -33,7 +33,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg init stg import -s ../patch-$(SNMPD_VERSION)/series - fakeroot debian/rules -j$(SONIC_CONFIG_MAKE_JOBS) binary + dpkg-buildpackage -rfakeroot -b -d -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/socat/Makefile b/src/socat/Makefile index 50c2c8d96d2..cc2cd723868 100644 --- a/src/socat/Makefile +++ b/src/socat/Makefile @@ -18,7 +18,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # Build source and Debian packages pushd socat-1.7.3.1 patch -p0 < ../enable_readline.patch - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd # Move the newly-built .deb packages to the destination directory diff --git a/src/sonic-device-data/Makefile b/src/sonic-device-data/Makefile index ecac92b00e0..ea4ca75d5b5 100644 --- a/src/sonic-device-data/Makefile +++ b/src/sonic-device-data/Makefile @@ -22,7 +22,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : done; # Build the package - dpkg-buildpackage -rfakeroot -b -us -uc + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd diff --git a/src/sonic-frr/Makefile b/src/sonic-frr/Makefile index 296f0226d2f..4bac1e42769 100644 --- a/src/sonic-frr/Makefile +++ b/src/sonic-frr/Makefile @@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : stg branch --create $(STG_BRANCH) $(FRR_TAG) stg import -s ../patch/series tools/tarsource.sh -V -e '-sonic' - dpkg-buildpackage -rfakeroot -b -us -uc -Ppkg.frr.nortrlib -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -Ppkg.frr.nortrlib -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) stg undo || true git clean -xfdf git checkout $(FRR_BRANCH) diff --git a/src/swig/Makefile b/src/swig/Makefile index f9deda6a831..2d3388eba13 100644 --- a/src/swig/Makefile +++ b/src/swig/Makefile @@ -13,7 +13,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : dpkg-source -x swig_$(SWIG_VERSION).dsc pushd ./swig-$(SWIG_VERSION) - dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/systemd-sonic-generator/Makefile b/src/systemd-sonic-generator/Makefile index 973c0d64eae..0f911ed8a4f 100644 --- a/src/systemd-sonic-generator/Makefile +++ b/src/systemd-sonic-generator/Makefile @@ -5,7 +5,7 @@ BINARY = systemd-sonic-generator MAIN_TARGET = $(BINARY)_1.0.0_$(CONFIGURED_ARCH).deb $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : - dpkg-buildpackage -us -uc -b + dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) mv ../$(MAIN_TARGET) $(DEST)/ rm ../$(BINARY)-* ../$(BINARY)_* diff --git a/src/tacacs/nss/Makefile b/src/tacacs/nss/Makefile index a0c42bd2f22..ca339e1cf67 100644 --- a/src/tacacs/nss/Makefile +++ b/src/tacacs/nss/Makefile @@ -24,7 +24,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : git $(GIT_APPLY) ../0004-Skip-accessing-tacacs-servers-for-local-non-tacacs-u.patch git $(GIT_APPLY) ../0005-libnss-Modify-parsing-of-IP-addr-and-port-number-str.patch - dpkg-buildpackage -rfakeroot -b -us -uc + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/tacacs/pam/Makefile b/src/tacacs/pam/Makefile index 487cf975fd7..c42ff59fcc1 100644 --- a/src/tacacs/pam/Makefile +++ b/src/tacacs/pam/Makefile @@ -9,7 +9,7 @@ DERIVED_TARGETS = libtac2_$(PAM_TACPLUS_VERSION)_$(CONFIGURED_ARCH).deb \ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # Obtain pam_tacplus rm -rf ./pam_tacplus - git clone https://github.com/jeroennijhof/pam_tacplus.git + git clone https://github.com/jeroennijhof/pam_tacplus.git pushd ./pam_tacplus git checkout -f v1.4.1 @@ -20,7 +20,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : git apply ../0004-management-vrf-support.patch git apply ../0005-pam-Modify-parsing-of-IP-address-and-port-number-to-.patch - dpkg-buildpackage -rfakeroot -b -us -uc + dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ diff --git a/src/thrift/Makefile b/src/thrift/Makefile index 6039b2e7cfa..2d1d4187135 100644 --- a/src/thrift/Makefile +++ b/src/thrift/Makefile @@ -26,7 +26,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% : # saithrift implementation relies on the bug in union serialization # (https://jira.apache.org/jira/browse/THRIFT-3650) patch -p1 < ../patch/0001-Revert-THRIFT-3650-incorrect-union-serialization.patch - CXXFLAGS="-DFORCE_BOOST_SMART_PTR" DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -d -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) + CXXFLAGS="-DFORCE_BOOST_SMART_PTR" DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -d -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR) popd mv $(DERIVED_TARGETS) $* $(DEST)/ From 6e40e0b9825357f3bab8452b007c66b2d8160a3c Mon Sep 17 00:00:00 2001 From: lguohan Date: Wed, 30 Dec 2020 04:57:44 -0800 Subject: [PATCH 3/5] [build]: setup -t option in docker run correctly (#6320) use bash -t test flag to check if input device is tty or not Signed-off-by: Guohan Lu --- Makefile.work | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.work b/Makefile.work index 323d05f027a..738c471269f 100644 --- a/Makefile.work +++ b/Makefile.work @@ -105,7 +105,7 @@ DOCKER_RUN := docker run --rm=true --privileged \ -w $(DOCKER_BUILDER_WORKDIR) \ -e "http_proxy=$(http_proxy)" \ -e "https_proxy=$(https_proxy)" \ - -i$(if $(TERM),t,) + -i$(shell { if [ -t 0 ]; then echo t; fi }) include rules/config From 2b9523a32085c5b054b7d6c5745db2d491816e68 Mon Sep 17 00:00:00 2001 From: Guohan Lu Date: Wed, 20 Jan 2021 01:06:40 -0800 Subject: [PATCH 4/5] [ci]: add azure pipeline yaml Signed-off-by: Guohan Lu --- .artifactignore | 5 +++ azure-pipelines.yml | 81 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 .artifactignore create mode 100644 azure-pipelines.yml diff --git a/.artifactignore b/.artifactignore new file mode 100644 index 00000000000..4e039ee7c04 --- /dev/null +++ b/.artifactignore @@ -0,0 +1,5 @@ +**/* +!target/*.bin +!target/*.log +!target/*.img.gz +!target/docker-sonic-vs.gz diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000000..53c90185882 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,81 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- main + +name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) + +resources: + repositories: + - repository: sonic-mgmt + type: github + name: Azure/sonic-mgmt + endpoint: build + +stages: +- stage: Build + pool: sonicbld + + jobs: + - job: + displayName: "broadcom" + timeoutInMinutes: 600 + steps: + - checkout: self + submodules: recursive + displayName: 'Checkout code' + + - script: | + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=rcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/broadcom" + make configure PLATFORM=broadcom + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-broadcom.bin + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.broadcom.201911 + displayName: "Archive sonic image" + + - job: + displayName: "mellanox" + timeoutInMinutes: 600 + steps: + - checkout: self + submodules: recursive + displayName: 'Checkout code' + + - script: | + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=rcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/mellanox" + make configure PLATFORM=mellanox + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS target/sonic-mellanox.bin + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.mellanox.201911 + displayName: "Archive sonic image" + + - job: + displayName: "kvm" + timeoutInMinutes: 600 + steps: + - checkout: self + submodules: recursive + displayName: 'Checkout code' + + - script: | + echo $(Build.BuildNumber) + sudo modprobe overlay + CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=rcache SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/vs" + make configure PLATFORM=vs + trap "sudo rm -rf fsroot" EXIT + make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \ + target/docker-sonic-vs.gz target/sonic-vs.img.gz && \ + sudo cp target/sonic-vs.img.gz /nfs/azpl/kvmimage/sonic-vs.$(Build.BuildNumber).img.gz + displayName: 'Build sonic image' + - publish: $(System.DefaultWorkingDirectory)/ + artifact: sonic-buildimage.kvm.201911 + displayName: "Archive sonic image" From f1e8eca679159ace7236f2fece935832e18be3eb Mon Sep 17 00:00:00 2001 From: Guohan Lu Date: Wed, 20 Jan 2021 08:53:36 -0800 Subject: [PATCH 5/5] [build]: make libsaivs-dev depends on libsaivs install libsaivs-dev will trigger install libsaivs Signed-off-by: Guohan Lu --- rules/sairedis.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/rules/sairedis.mk b/rules/sairedis.mk index fd026541a4f..13f73eff4e8 100644 --- a/rules/sairedis.mk +++ b/rules/sairedis.mk @@ -22,6 +22,7 @@ $(eval $(call add_derived_package,$(LIBSAIREDIS),$(LIBSAIVS))) LIBSAIVS_DEV = libsaivs-dev_1.0.0_$(CONFIGURED_ARCH).deb $(eval $(call add_derived_package,$(LIBSAIREDIS),$(LIBSAIVS_DEV))) +$(LIBSAIVS_DEV)_DEPENDS += $(LIBSAIVS) ifneq ($(CONFIGURED_PLATFORM),vs) SYNCD = syncd_1.0.0_$(CONFIGURED_ARCH).deb