Skip to content

Commit b41944a

Browse files
committed
[build]: fix dpkg admindir corruption issue in parallel build (sonic-net#6408)
Fix sonic-net#119 when parallel build is enable, multiple dpkg-buildpackage instances are running at the same time. /var/lib/dpkg is shared by all instances and the /var/lib/dpkg/updates could be corrupted and cause the build failure. the fix is to use overlay fs to mount separate /var/lib/dpkg for each dpkg-buildpackage instance so that they are not affecting each other. Signed-off-by: Guohan Lu <[email protected]>
1 parent 50550c1 commit b41944a

32 files changed

Lines changed: 49 additions & 33 deletions

File tree

rules/functions

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,3 +96,15 @@ endef
9696
###############################################################################
9797

9898
expand = $(foreach d,$(1),$(call expand,$($(d)_$(2)),$(2))) $(1)
99+
100+
###############################################################################
101+
## Setup overlay fs for dpkg admin directory /var/lib/dpkg
102+
###############################################################################
103+
define SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR
104+
upperdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
105+
workdir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
106+
mergedir=$(shell mktemp -d -p $(DPKG_ADMINDIR_PATH))
107+
sudo mount -t overlay overlay -olowerdir=/var/lib/dpkg,upperdir=$$upperdir,workdir=$$workdir $$mergedir
108+
export SONIC_DPKG_ADMINDIR=$$mergedir
109+
trap "sudo umount $$mergedir && rm -rf $$mergedir $$upperdir $$workdir" EXIT
110+
endef

slave.mk

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ STRETCH_DEBS_PATH = $(TARGET_PATH)/debs/stretch
3838
STRETCH_FILES_PATH = $(TARGET_PATH)/files/stretch
3939
DBG_IMAGE_MARK = dbg
4040
DBG_SRC_ARCHIVE_FILE = $(TARGET_PATH)/sonic_src.tar.gz
41+
DPKG_ADMINDIR_PATH = /sonic/dpkg
4142

4243
CONFIGURED_PLATFORM := $(shell [ -f .platform ] && cat .platform || echo generic)
4344
PLATFORM_PATH = platform/$(CONFIGURED_PLATFORM)
@@ -68,6 +69,7 @@ configure :
6869
@mkdir -p target/files/stretch
6970
@mkdir -p target/python-debs
7071
@mkdir -p target/python-wheels
72+
@mkdir -p $(DPKG_ADMINDIR_PATH)
7173
@echo $(PLATFORM) > .platform
7274
@echo $(PLATFORM_ARCH) > .arch
7375

@@ -331,6 +333,7 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
331333
rm -f $(addprefix $(DEBS_PATH)/, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS))
332334
# Apply series of patches if exist
333335
if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && QUILT_PATCHES=../$(notdir $($*_SRC_PATH)).patch quilt push -a; popd; fi
336+
$(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR)
334337
# Build project and take package
335338
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC}" make DEST=$(shell pwd)/$(DEBS_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(DEBS_PATH)/$* $(LOG)
336339
# Clean up
@@ -354,9 +357,10 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
354357
# Build project
355358
pushd $($*_SRC_PATH) $(LOG)
356359
[ ! -f ./autogen.sh ] || ./autogen.sh $(LOG)
360+
$(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR)
357361
$(if $($*_DPKG_TARGET),
358-
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) $(LOG),
359-
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) $(LOG)
362+
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) --admindir $$mergedir $(LOG),
363+
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $$mergedir $(LOG)
360364
)
361365
popd $(LOG)
362366
# Clean up

src/bash/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
1010
dget -u https://launchpad.net/debian/+archive/primary/+sourcefiles/bash/$(BASH_VERSION_FULL)/bash_$(BASH_VERSION_FULL).dsc
1111

1212
pushd bash-$(BASH_VERSION_MAJOR)
13-
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS)
13+
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
1414
popd
1515

1616
mv $* $(DEST)/

src/hiredis/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
1414

1515
dpkg-source -x hiredis_$(HIREDIS_VERSION_FULL).dsc
1616
pushd hiredis-$(HIREDIS_VERSION)
17-
fakeroot debian/rules -j$(SONIC_CONFIG_MAKE_JOBS) binary
17+
dpkg-buildpackage -rfakeroot -d -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
1818
popd
1919

2020
mv $* $(DERIVED_TARGETS) $(DEST)/

src/ifupdown2/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
1414
pushd ./ifupdown2-$(IFUPDOWN2_VERSION)
1515

1616
# Build source and Debian packages
17-
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
17+
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
1818
popd
1919

2020
# Move the newly-built .deb packages to the destination directory

src/initramfs-tools/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
1919

2020
# Build the package
2121
rm -f debian/*.debhelper.log
22-
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
22+
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
2323
popd
2424

2525
mv $(DERIVED_TARGETS) $* $(DEST)/

src/iproute2/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
1717
dpkg-source -x iproute2_$(IPROUTE2_VERSION_FULL).dsc
1818

1919
pushd iproute2-$(IPROUTE2_VERSION)
20-
dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS)
20+
dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
2121
popd
2222

2323
mv $* $(DEST)/

src/iptables/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
3838
stg import -s ../patch/series
3939

4040
# Build source and Debian packages
41-
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
41+
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
4242
popd
4343

4444
# Move the newly-built .deb packages to the destination directory

src/isc-dhcp/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
2323
stg import -s ../patch/series
2424

2525
# Build source and Debian packages
26-
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
26+
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
2727
popd
2828

2929
# Move the newly-built .deb packages to the destination directory

src/libnl3/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
2121
git checkout tags/libnl$(subst .,_,$(LIBNL3_VERSION_BASE))
2222

2323
ln -s ../debian debian
24-
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS)
24+
dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
2525
popd
2626

2727
mv $(DERIVED_TARGETS) $* $(DEST)/

0 commit comments

Comments
 (0)