Support SONiC Reproduceable Build

Author: xumia

.gitignore

@@ -20,6 +20,7 @@ target/
 *-dbg
 *dbg.j2
 *.img
+**/buildinfo
 
 # Autogenerated Dockerfiles
 sonic-slave*/Dockerfile

Makefile.work

@@ -89,10 +89,18 @@ else
 SLAVE_DIR = sonic-slave-jessie
 endif
 
-SLAVE_BASE_TAG = $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile && sha1sum $(SLAVE_DIR)/Dockerfile | awk '{print substr($$1,0,11);}')
-SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}')
+include rules/config
+
 SLAVE_BASE_IMAGE = $(SLAVE_DIR)
 SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER)
+$(shell SONIC_ENABLE_VERSION_CONTROL=$(SONIC_ENABLE_VERSION_CONTROL) \
+    SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
+    TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \
+    scripts/generate_buildinfo_config.sh)
+$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
+$(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
+SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}')
+SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}')
 
 OVERLAY_MODULE_CHECK := \
     lsmod | grep -q "^overlay " &>/dev/null || \
@@ -118,8 +126,6 @@ DOCKER_RUN := docker run --rm=true --privileged --init \
     -i$(if $(TERM),t,) \
     $(SONIC_BUILDER_EXTRA_CMDLINE)
 
-include rules/config
-
 ifneq ($(DOCKER_BUILDER_USER_MOUNT),)
 	DOCKER_RUN += $(foreach mount,$(subst $(comma), ,$(DOCKER_BUILDER_USER_MOUNT)), $(addprefix -v , $(mount)))
 endif
@@ -172,7 +178,8 @@ DOCKER_BASE_BUILD = docker build --no-cache \
 		    -t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
 		    --build-arg http_proxy=$(http_proxy) \
 		    --build-arg https_proxy=$(https_proxy) \
-		    $(SLAVE_DIR)
+		    $(SLAVE_DIR); \
+		    scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target
 
 DOCKER_BUILD = docker build --no-cache \
 	       --build-arg user=$(USER) \
@@ -217,7 +224,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            TELEMETRY_WRITABLE=$(TELEMETRY_WRITABLE) \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \
-                           SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
+                           SLAVE_DIR=$(SLAVE_DIR) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset
@@ -242,12 +249,12 @@ endif
 	    $(DOCKER_BUILD) ; }
 ifeq "$(KEEP_SLAVE_ON)" "yes"
     ifdef SOURCE_FOLDER
-		@$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; /bin/bash"
+		@$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@;  scripts/collect_build_version_files.sh $$?; /bin/bash"
     else
-		@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; /bin/bash"
+		@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh $$?; /bin/bash"
     endif
 else
-	@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) $(SONIC_BUILD_INSTRUCTION) $@
+	@$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh $$?"
 endif
 
 sonic-slave-base-build :

build_debian.sh

@@ -75,16 +75,16 @@ pushd $FILESYSTEM_ROOT
 sudo mount --bind . .
 popd
 
-## Build a basic Debian system by debootstrap
-echo '[INFO] Debootstrap...'
-if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
-    # qemu arm bin executable for cross-building
-    sudo mkdir -p $FILESYSTEM_ROOT/usr/bin
-    sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true
-    sudo http_proxy=$http_proxy debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian
-else
-    sudo http_proxy=$http_proxy debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian
-fi
+## Build the host base debian system
+echo '[INFO] Build host base image...'
+TARGET_PATH=$TARGET_PATH scripts/build_host_base_image.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT
+
+# Prepare buildinfo
+sudo scripts/prepare_base_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy
+
+# Generate version files for apt/pip/pip3 packages
+# sudo LANG=C chroot $FILESYSTEM_ROOT generate_version_files
+
 
 ## Config hostname and hosts, otherwise 'sudo ...' will complain 'sudo: unable to resolve host ...'
 sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo '$HOSTNAME' > /etc/hostname"
@@ -568,6 +568,8 @@ sudo du -hsx $FILESYSTEM_ROOT
 sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker
 sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR
 
+scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
+
 ## Compress docker files
 pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd
 

dockers/docker-base-buster/Dockerfile.j2

@@ -7,6 +7,9 @@ FROM multiarch/debian-debootstrap:arm64-buster
 FROM debian:buster
 {% endif %}
 
+RUN echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \
+    echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list
+
 # Clean documentation in FROM image
 RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true
 

files/build/scripts/apt-get

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+INSTALL=
+
+VERSION_FILE="/usr/local/share/buildinfo/base-versions/versions-deb"
+for para in $@
+do
+    if [[ "$para" != -* ]]; then
+        continue
+    fi
+    if [ ! -z "$INSTALL" ]; then
+        if [[ "$para" == *=* ]]; then
+            continue
+            package=$(echo "$para" | cut -d= -f1)
+            version=$(echo "$para" | cut -d= -f2)
+            if grep "^${package}=" $VERSION_FILE; then
+                if ! grep "^${package}==${version}$" $VERSION_FILE; then
+                    EXPECTED_VERSION=$(grep "^${package}=" $VERSION_FILE | cut -d= -f3)
+                    echo "${package}=${version}, expected version is ${EXPECTED_VERSION}"
+                    exit 1
+                fi
+            fi
+        else
+            if ! grep "^${package}=" $VERSION_FILE; then
+                echo "The version of the package ${package} is not specified."
+                exit 1
+            fi
+        fi
+    elif [[ "$para" == "install"  ]]; then
+        INSTALL=y
+    fi
+done
+
+
+/usr/bin/apt-get $@

files/build/scripts/apt-get-version

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+#apt list --installed | sed "1d" | awk '{gsub("/.*", "", $1); print $1"="$2}'
+dpkg-query -W -f '${Package}==${Version}\n'

files/build/scripts/clean_buildinfo.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+rm -f /usr/local/sbin/apt-get
+rm -f /usr/local/sbin/pip
+rm -f /usr/local/sbin/pip3
+rm -f /usr/local/sbin/wget

files/build/scripts/collect_version_files

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+TARGET_PATH=$1
+
+mkdir -p $TARGET_PATH
+dpkg-query -W -f '${Package}==${Version}\n' > "${TARGET_PATH}/versions-deb"
+([ -x "/usr/local/bin/pip" ] || [ -x "/usr/bin/pip" ]) && pip freeze > "${TARGET_PATH}/versions-py2"
+([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze > "${TARGET_PATH}/versions-py3"
+
+exit 0

files/build/scripts/curl

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+
+PARENT_PATH=$(dirname $0)
+REAL_COMMAND=/usr/bin/curl
+
+REAL_COMMAND=$REAL_COMMAND ${PARENT_PATH}/wget $@

files/build/scripts/pip

@@ -0,0 +1,48 @@
+#!/bin/bash
+  
+. /usr/local/share/buildinfo/config/buildinfo.config
+
+[ -z "$PIP_COMPONENT" ] && PIP_COMPONENT=py2
+
+if [ "$PIP_COMPONENT" == "py2" ]; then
+    PIP_COMMAND=/usr/local/bin/pip
+    [ ! -f $PIP_COMMAND ] && PIP_COMMAND=/usr/bin/pip
+else
+    PIP_COMMAND=/usr/bin/pip3
+fi
+
+VERSION_FILE="$BUILDINFO_PATH/versions/versions-${PIP_COMPONENT}"
+ENABLE_VERSION_CONTROL=$(check_version_control "$PIP_COMPONENT")
+
+
+if [ "$ENABLE_VERSION_CONTROL" != "y" ]; then
+    $PIP_COMMAND "$@"
+    exit $?
+fi
+
+paras=("$@")
+FOUND=false
+INSTALL=false
+VERSION_CONFIG_FILE="${VERSION_FILE}.config"
+cp -f $VERSION_FILE $VERSION_CONFIG_FILE
+for para in "${paras[@]}"
+do
+    ([ "$para" == "-c" ] || [ "$para" == "--constraint" ]) && FOUND=true
+    [ "$para" == "install" ] && INSTALL=true
+    if [[ "$para" == *.whl ]]; then
+        package_name=$(echo $para | cut -d- -f1 | tr _ .)
+        sed "/^${package_name}==/d" -i $VERSION_CONFIG_FILE
+    fi
+done
+
+if [ "$SONIC_ENABLE_VERSION_CONTROL" == "y" ] && [ "$FOUND" == "false" ] && [ "$INSTALL" == "true" ]; then
+    paras+=("-c")
+    paras+=("${VERSION_CONFIG_FILE}")
+fi
+
+
+if [ ! -x "$PIP_COMMAND" ] && [ " $1" == "freeze" ]; then
+    exit 1
+fi
+
+$PIP_COMMAND ${paras[@]}