From 0bd96d84773af46e3f09bc3b2825f1bcc104041b Mon Sep 17 00:00:00 2001 From: xumia Date: Mon, 2 Nov 2020 23:40:54 +0000 Subject: [PATCH 01/18] Support SONiC reproduceable build for deb/py2/py3/web --- .gitignore | 1 + Makefile | 3 + Makefile.work | 23 +- build_debian.sh | 22 +- dockers/docker-base-buster/Dockerfile.j2 | 3 + files/build/scripts/apt-get | 35 + files/build/scripts/collect_version_files | 13 + files/build/scripts/curl | 7 + files/build/scripts/pip | 57 ++ files/build/scripts/pip2 | 13 + files/build/scripts/pip3 | 13 + files/build/scripts/post_run_buildinfo | 22 + files/build/scripts/pre_run_buildinfo | 21 + files/build/scripts/wget | 96 +++ files/build/templates/buildinfo.config.j2 | 37 + .../templates/dockerfile_auto_generate.j2 | 13 + files/build/versions/default/readme.md | 1 + rules/config | 17 + scripts/build_host_base_image.sh | 88 +++ scripts/collect_build_version_files.sh | 28 + scripts/collect_docker_version_files.sh | 21 + scripts/collect_host_image_version_files.sh | 11 + scripts/generate_buildinfo_config.sh | 26 + scripts/prepare_base_image_buildinfo.sh | 35 + scripts/prepare_docker_buildinfo.sh | 59 ++ scripts/prepare_slave_container_buildinfo.sh | 16 + scripts/versions_manager.py | 640 ++++++++++++++++++ slave.mk | 23 + sonic-slave-buster/Dockerfile.j2 | 13 +- sonic-slave-stretch/Dockerfile.j2 | 14 +- 30 files changed, 1347 insertions(+), 24 deletions(-) create mode 100755 files/build/scripts/apt-get create mode 100755 files/build/scripts/collect_version_files create mode 100755 files/build/scripts/curl create mode 100755 files/build/scripts/pip create mode 100755 files/build/scripts/pip2 create mode 100755 files/build/scripts/pip3 create mode 100755 files/build/scripts/post_run_buildinfo create mode 100755 files/build/scripts/pre_run_buildinfo create mode 100755 files/build/scripts/wget create mode 100644 files/build/templates/buildinfo.config.j2 create mode 100644 files/build/templates/dockerfile_auto_generate.j2 create mode 100644 files/build/versions/default/readme.md create mode 100755 scripts/build_host_base_image.sh create mode 100755 scripts/collect_build_version_files.sh create mode 100755 scripts/collect_docker_version_files.sh create mode 100755 scripts/collect_host_image_version_files.sh create mode 100755 scripts/generate_buildinfo_config.sh create mode 100755 scripts/prepare_base_image_buildinfo.sh create mode 100755 scripts/prepare_docker_buildinfo.sh create mode 100755 scripts/prepare_slave_container_buildinfo.sh create mode 100755 scripts/versions_manager.py diff --git a/.gitignore b/.gitignore index eda891212d5..111e73c2745 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ target/ *-dbg *dbg.j2 *.img +**/buildinfo # Autogenerated Dockerfiles sonic-slave*/Dockerfile diff --git a/Makefile b/Makefile index b855017f928..8bb8f443aef 100644 --- a/Makefile +++ b/Makefile @@ -34,3 +34,6 @@ ifeq ($(NOSTRETCH), 0) BLDENV=stretch make -f Makefile.work $@ endif BLDENV=buster make -f Makefile.work $@ + +freeze: + @scripts/versions_manager.py freeze $(OPTIONS) diff --git a/Makefile.work b/Makefile.work index 79f521488fe..e244d09728f 100644 --- a/Makefile.work +++ b/Makefile.work @@ -89,10 +89,17 @@ else SLAVE_DIR = sonic-slave-jessie endif -SLAVE_BASE_TAG = $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile && sha1sum $(SLAVE_DIR)/Dockerfile | awk '{print substr($$1,0,11);}') -SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile | sha1sum | awk '{print substr($$1,0,11);}') +include rules/config + SLAVE_BASE_IMAGE = $(SLAVE_DIR) SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER) +$(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + scripts/generate_buildinfo_config.sh) +$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile) +$(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV)) +SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}') +SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}') OVERLAY_MODULE_CHECK := \ lsmod | grep -q "^overlay " &>/dev/null || \ @@ -118,8 +125,6 @@ DOCKER_RUN := docker run --rm=true --privileged --init \ -i$(if $(TERM),t,) \ $(SONIC_BUILDER_EXTRA_CMDLINE) -include rules/config - ifneq ($(DOCKER_BUILDER_USER_MOUNT),) DOCKER_RUN += $(foreach mount,$(subst $(comma), ,$(DOCKER_BUILDER_USER_MOUNT)), $(addprefix -v , $(mount))) endif @@ -172,7 +177,8 @@ DOCKER_BASE_BUILD = docker build --no-cache \ -t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \ --build-arg http_proxy=$(http_proxy) \ --build-arg https_proxy=$(https_proxy) \ - $(SLAVE_DIR) + $(SLAVE_DIR); \ + scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target DOCKER_BUILD = docker build --no-cache \ --build-arg user=$(USER) \ @@ -218,6 +224,7 @@ SONIC_BUILD_INSTRUCTION := make \ EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \ BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \ SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \ + SLAVE_DIR=$(SLAVE_DIR) \ $(SONIC_OVERRIDE_BUILD_VARS) .PHONY: sonic-slave-build sonic-slave-bash init reset @@ -242,12 +249,12 @@ endif $(DOCKER_BUILD) ; } ifeq "$(KEEP_SLAVE_ON)" "yes" ifdef SOURCE_FOLDER - @$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; /bin/bash" + @$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh $$?; /bin/bash" else - @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; /bin/bash" + @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh $$?; /bin/bash" endif else - @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) $(SONIC_BUILD_INSTRUCTION) $@ + @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh $$?" endif sonic-slave-base-build : diff --git a/build_debian.sh b/build_debian.sh index a0d519d7a31..1f1b6c4ec8d 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -70,16 +70,16 @@ pushd $FILESYSTEM_ROOT sudo mount --bind . . popd -## Build a basic Debian system by debootstrap -echo '[INFO] Debootstrap...' -if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then - # qemu arm bin executable for cross-building - sudo mkdir -p $FILESYSTEM_ROOT/usr/bin - sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true - sudo http_proxy=$http_proxy debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian -else - sudo http_proxy=$http_proxy debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian -fi +## Build the host base debian system +echo '[INFO] Build host base image...' +TARGET_PATH=$TARGET_PATH scripts/build_host_base_image.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT + +# Prepare buildinfo +sudo scripts/prepare_base_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy + +# Generate version files for apt/pip/pip3 packages +# sudo LANG=C chroot $FILESYSTEM_ROOT generate_version_files + ## Config hostname and hosts, otherwise 'sudo ...' will complain 'sudo: unable to resolve host ...' sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo '$HOSTNAME' > /etc/hostname" @@ -571,6 +571,8 @@ sudo du -hsx $FILESYSTEM_ROOT sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR +scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT + ## Compress docker files pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2 index bf1d8197bf4..c6d8329b5b8 100644 --- a/dockers/docker-base-buster/Dockerfile.j2 +++ b/dockers/docker-base-buster/Dockerfile.j2 @@ -7,6 +7,9 @@ FROM multiarch/debian-debootstrap:arm64-buster FROM debian:buster {% endif %} +RUN echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list + # Clean documentation in FROM image RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true diff --git a/files/build/scripts/apt-get b/files/build/scripts/apt-get new file mode 100755 index 00000000000..4fe2659be19 --- /dev/null +++ b/files/build/scripts/apt-get @@ -0,0 +1,35 @@ +#!/bin/bash + +INSTALL= + +VERSION_FILE="/usr/local/share/buildinfo/base-versions/versions-deb" +for para in $@ +do + if [[ "$para" != -* ]]; then + continue + fi + if [ ! -z "$INSTALL" ]; then + if [[ "$para" == *=* ]]; then + continue + package=$(echo "$para" | cut -d= -f1) + version=$(echo "$para" | cut -d= -f2) + if grep "^${package}=" $VERSION_FILE; then + if ! grep "^${package}==${version}$" $VERSION_FILE; then + EXPECTED_VERSION=$(grep "^${package}=" $VERSION_FILE | cut -d= -f3) + echo "${package}=${version}, expected version is ${EXPECTED_VERSION}" + exit 1 + fi + fi + else + if ! grep "^${package}=" $VERSION_FILE; then + echo "The version of the package ${package} is not specified." + exit 1 + fi + fi + elif [[ "$para" == "install" ]]; then + INSTALL=y + fi +done + + +/usr/bin/apt-get $@ diff --git a/files/build/scripts/collect_version_files b/files/build/scripts/collect_version_files new file mode 100755 index 00000000000..fed1663e783 --- /dev/null +++ b/files/build/scripts/collect_version_files @@ -0,0 +1,13 @@ +#!/bin/bash + +TARGET_PATH=$1 +ARCH=$(dpkg --print-architecture) +DIST=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) +([ -z "$DIST" ] && grep -q jessie /etc/os-release) && DIST=jessie + +mkdir -p $TARGET_PATH +dpkg-query -W -f '${Package}==${Version}\n' > "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" +([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze > "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" +([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze > "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" + +exit 0 diff --git a/files/build/scripts/curl b/files/build/scripts/curl new file mode 100755 index 00000000000..b13573535b8 --- /dev/null +++ b/files/build/scripts/curl @@ -0,0 +1,7 @@ +#!/bin/bash + + +PARENT_PATH=$(dirname $0) +REAL_COMMAND=/usr/bin/curl + +REAL_COMMAND=$REAL_COMMAND ${PARENT_PATH}/wget $@ diff --git a/files/build/scripts/pip b/files/build/scripts/pip new file mode 100755 index 00000000000..cbc004650bc --- /dev/null +++ b/files/build/scripts/pip @@ -0,0 +1,57 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/config/buildinfo.config + + +if [ -z "$PIP_COMMAND" ]; then + [ -x /usr/bin/pip ] && PIP_COMMAND=/usr/bin/pip + [ -x /usr/local/bin/pip ] && PIP_COMMAND=/usr/local/bin/pip +fi + +if [ ! -x "$PIP_COMMAND" ]; then + echo "The command pip not found" 1>&2 + exit 1 +fi + +if [ -z "$PIP_COMPONENT" ]; then + PIP_COMPONENT=py2 + if $PIP_COMMAND --version | grep -q "python 3"; then + PIP_COMPONENT=py3 + fi +fi + +ENABLE_VERSION_CONTROL=ENABLE_VERSION_CONTROL_PY2 +[ "$PIP_COMPONENT" == "py3" ] && ENABLE_VERSION_CONTROL=ENABLE_VERSION_CONTROL_PY3 +VERSION_FILE="$BUILDINFO_PATH/versions/versions-${PIP_COMPONENT}" + +if [ "$ENABLE_VERSION_CONTROL" != "y" ]; then + $PIP_COMMAND "$@" + exit $? +fi + +paras=("$@") +FOUND=false +INSTALL=false +VERSION_CONFIG_FILE="${VERSION_FILE}.config" +cp -f $VERSION_FILE $VERSION_CONFIG_FILE +for para in "${paras[@]}" +do + ([ "$para" == "-c" ] || [ "$para" == "--constraint" ]) && FOUND=true + [ "$para" == "install" ] && INSTALL=true + if [[ "$para" == *.whl ]]; then + package_name=$(echo $para | cut -d- -f1 | tr _ .) + sed "/^${package_name}==/d" -i $VERSION_CONFIG_FILE + fi +done + +if [ "$FOUND" == "false" ] && [ "$INSTALL" == "true" ]; then + paras+=("-c") + paras+=("${VERSION_CONFIG_FILE}") +fi + + +if [ ! -x "$PIP_COMMAND" ] && [ " $1" == "freeze" ]; then + exit 1 +fi + +$PIP_COMMAND ${paras[@]} diff --git a/files/build/scripts/pip2 b/files/build/scripts/pip2 new file mode 100755 index 00000000000..42c65865fdf --- /dev/null +++ b/files/build/scripts/pip2 @@ -0,0 +1,13 @@ +#!/bin/bash + +PARENT_PATH=$(dirname $0) + +PIP_COMMAND=/usr/bin/pip2 +[ -x /usr/local/bin/pip2 ] && PIP_COMMAND=/usr/local/bin/pip2 + +if [ ! -x "$PIP_COMMAND" ]; then + echo "The command pip2 not found" 1>&2 + exit 1 +fi + +PIP_COMPONENT=py2 PIP_COMMAND="$PIP_COMMAND" ${PARENT_PATH}/pip $@ diff --git a/files/build/scripts/pip3 b/files/build/scripts/pip3 new file mode 100755 index 00000000000..15788a2e1e4 --- /dev/null +++ b/files/build/scripts/pip3 @@ -0,0 +1,13 @@ +#!/bin/bash + +PARENT_PATH=$(dirname $0) + +PIP_COMMAND=/usr/bin/pip3 +[ -x /usr/local/bin/pip3 ] && PIP_COMMAND=/usr/local/bin/pip3 + +if [ ! -x "$PIP_COMMAND" ]; then + echo "The command pip3 not found" 1>&2 + exit 1 +fi + +PIP_COMPONENT=py2 PIP_COMMAND="$PIP_COMMAND" ${PARENT_PATH}/pip $@ diff --git a/files/build/scripts/post_run_buildinfo b/files/build/scripts/post_run_buildinfo new file mode 100755 index 00000000000..c9de5341cc9 --- /dev/null +++ b/files/build/scripts/post_run_buildinfo @@ -0,0 +1,22 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/config/buildinfo.config + +[ -d $POST_VERSION_PATH ] && rm -rf $POST_VERSION_PATH + +collect_version_files $POST_VERSION_PATH +chmod -R a+rw $BUILDINFO_PATH + +for file in $POST_VERSION_PATH/versions-* +do + filename=$(basename $file) + init_file=$PRE_VERSION_PATH/$filename + diff_file=$DIFF_VERSION_PATH/$filename + cat $file | sort > $diff_file + if [ -f $init_file ]; then + cat $file $init_file $init_file | sort | LC_ALL=C uniq -u > $diff_file + fi +done + +[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $DIFF_VERSION_PATH/ +rm -rf $BUILD_VERSION_PATH/* diff --git a/files/build/scripts/pre_run_buildinfo b/files/build/scripts/pre_run_buildinfo new file mode 100755 index 00000000000..9299b37802f --- /dev/null +++ b/files/build/scripts/pre_run_buildinfo @@ -0,0 +1,21 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/config/buildinfo.config + +[ -d $DIFF_VERSION_PATH ] && rm -rf $DIFF_VERSION_PATH +mkdir -p $DIFF_VERSION_PATH +mkdir -p $BUILD_VERSION_PATH +mkdir -p $LOG_PATH + +[ -d $PRE_VERSION_PATH ] && rm -rf $PRE_VERSION_PATH +collect_version_files $PRE_VERSION_PATH + +chmod -R a+rw $BUILDINFO_PATH + +if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]; then + cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ +fi + +cp -rf $BUILDINFO_PATH/trusted.gpg.d/* /etc/apt/trusted.gpg.d/ + +exit 0 diff --git a/files/build/scripts/wget b/files/build/scripts/wget new file mode 100755 index 00000000000..b78806176d6 --- /dev/null +++ b/files/build/scripts/wget @@ -0,0 +1,96 @@ +#!/bin/bash + +. /usr/local/share/buildinfo/config/buildinfo.config +COMPONENT=web +VERSION_FILE=$VERSION_PATH/versions-web +BUILD_VERSION_FILE=$BUILD_VERSION_PATH/versions-web +ENABLE_VERSION_CONTROL=$(check_version_control "$COMPONENT") + +[ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/wget +PARAS=("$@") + +if [ "$SKIP_BUILD_HOOK" == "y" ]; then + $REAL_COMMAND "$@" + exit $? +fi + +get_version() +{ + local package_url=$1 + if [[ "$REAL_COMMAND" == */curl ]]; then + $REAL_COMMAND "$package_url" | md5sum | cut -d' ' -f1 + else + $REAL_COMMAND -qO - "$package_url" | md5sum | cut -d' ' -f1 + fi +} + +get_weburl_index() +{ + for (( i=0; i<${#PARAS[@]}; i++ )) + do + local para=${PARAS[$i]} + if [[ "$para" == *://* ]]; then + echo $i + return + fi + done +} + +check_if_file_exist() +{ + if [[ "$REAL_COMMAND" == */curl ]]; then + $REAL_COMMAND --output /dev/null --silent --head --fail "$1" > /dev/null 2>&1 + else + $REAL_COMMAND -S --spider $1 2>&1 | grep -q "200 OK" + fi +} + +process() +{ + local index=$(get_weburl_index) + + if [ -z "$index" ]; then + log_err "Failed to parse the url" + return 1 + fi + local package_url=${PARAS[$index]} + local version="" + local real_version="" + [ -f $VERSION_FILE ] && version=$(grep "^${package_url}=" $VERSION_FILE | awk -F"==" '{print $NF}') + + if [ ! -z "$version" ]; then + local filename=$(echo $package_url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) + local version_filename="${filename}-${version}" + local url="${PACKAGE_URL_PREFIX}${version_filename}" + PARAS[$index]=$url + + # Try to download package from the proxy + if check_if_file_exist $url; then + ${REAL_COMMAND} ${PARAS[@]} + local result=$? + [ -f "$version_filename" ] && mv -f "${version_filename}" "${filename}" + return $result + fi + + real_version=$(get_version "$package_url") + if [ "$ENABLE_VERSION_CONTROL" == "y" ] && [ "$version" != "$real_version" ]; then + return 1 + fi + elif [ "$ENABLE_VERSION_CONTROL" == "y" ]; then + return 1 + fi + + [ -z "$real_version" ] && real_version=$(get_version "$package_url") + mkdir -p $LOG_PATH > /dev/null 2>&1 + echo "$package_url==$real_version" >> ${BUILD_VERSION_FILE} + + # If the version is mismatched, exit the wget/curl + if [ "$ENABLE_VERSION_CONTROL" == "y" ] && [ "$version" != "$real_version" ]; then + return 1 + fi + + ${REAL_COMMAND} "$@" +} + + +process "$@" diff --git a/files/build/templates/buildinfo.config.j2 b/files/build/templates/buildinfo.config.j2 new file mode 100644 index 00000000000..ee0d3d64a37 --- /dev/null +++ b/files/build/templates/buildinfo.config.j2 @@ -0,0 +1,37 @@ +#!/bin/bash + +BUILDINFO_PATH=/usr/local/share/buildinfo + +LOG_PATH=$BUILDINFO_PATH/log +VERSION_PATH=$BUILDINFO_PATH/versions +PRE_VERSION_PATH=$BUILDINFO_PATH/pre-versions +DIFF_VERSION_PATH=$BUILDINFO_PATH/diff-versions +BUILD_VERSION_PATH=$BUILDINFO_PATH/build-versions +POST_VERSION_PATH=$BUILDINFO_PATH/post-versions +VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb + +PACKAGE_URL_PREFIX={{ PACKAGE_URL_PREFIX }} +SONIC_VERSION_CONTROL_COMPONENTS={{ SONIC_VERSION_CONTROL_COMPONENTS }} +URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#") + +log_err() +{ + echo "$1" >> $LOG_PATH/error.log + echo "$1" 1>&2 +} + +check_version_control() +{ + if [[ ",$SONIC_VERSION_CONTROL_COMPONENTS," == *,all,* ]] || [[ ",$SONIC_VERSION_CONTROL_COMPONENTS," == *,$1,* ]]; then + echo "y" + else + echo "n" + fi +} + +ENABLE_VERSION_CONTROL_DEB=$(check_version_control "deb") +ENABLE_VERSION_CONTROL_PY2=$(check_version_control "py2") +ENABLE_VERSION_CONTROL_PY3=$(check_version_control "py3") +ENABLE_VERSION_CONTROL_WEB=$(check_version_control "web") +ENABLE_VERSION_CONTROL_GIT=$(check_version_control "git") +ENABLE_VERSION_CONTROL_DCK=$(check_version_control "dck") diff --git a/files/build/templates/dockerfile_auto_generate.j2 b/files/build/templates/dockerfile_auto_generate.j2 new file mode 100644 index 00000000000..8fc473583d8 --- /dev/null +++ b/files/build/templates/dockerfile_auto_generate.j2 @@ -0,0 +1,13 @@ +{% if generate_code == 'before_run' %} +# ---- Begin Auto-Generated Code ---- +COPY ["buildinfo", "/usr/local/share/buildinfo"] +ENV OLDPATH=$PATH +ENV PATH="/usr/local/share/buildinfo/scripts:$PATH" +RUN pre_run_buildinfo +# --- End Auto-Generated Code ---- +{% elif generate_code == 'after_run' %} +# ---- Begin Auto-Generated Code ---- +RUN post_run_buildinfo +{% if env("set_env_path", default='y') == 'y' %}ENV PATH=$OLDPATH{% endif %} +# ---- End Auto-Generated Code ---- +{% endif %} diff --git a/files/build/versions/default/readme.md b/files/build/versions/default/readme.md new file mode 100644 index 00000000000..438972e1906 --- /dev/null +++ b/files/build/versions/default/readme.md @@ -0,0 +1 @@ +The default versions diff --git a/rules/config b/rules/config index 8f486d8170e..611f7ca9889 100644 --- a/rules/config +++ b/rules/config @@ -162,3 +162,20 @@ K8s_GCR_IO_PAUSE_VERSION = 3.2 # CA_CERT = # The relative path is build root folder. SONIC_ENABLE_IMAGE_SIGNATURE ?= n + +# PACKAGE_URL_PREFIX - the package url prefix +PACKAGE_URL_PREFIX ?= https://packages.trafficmanager.net/public/packages + +# TRUSTED_GPG_URLS - the trusted gpgs, separated by comma +TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,https://packages.microsoft.com/keys/microsoft.asc + +# SONIC_VERSION_CONTROL_COMPONENTS - Valid values: none|all|components..., the components consist of one or multiple: deb,py2,py3,web,git,dck, seperated by comma +# none : disable the version control +# all : enable the version control for all components +# deb : debian packages +# py2 : python2 packages +# py3 : python3 pakcages +# web : web packages, downloaded by wget, curl +# git : git repositories, donloaded by git clone +# dck : docker base images +SONIC_VERSION_CONTROL_COMPONENTS ?= none diff --git a/scripts/build_host_base_image.sh b/scripts/build_host_base_image.sh new file mode 100755 index 00000000000..dca4d173a9c --- /dev/null +++ b/scripts/build_host_base_image.sh @@ -0,0 +1,88 @@ +#!/bin/bash + + +CONFIGURED_ARCH=$1 +IMAGE_DISTRO=$2 +FILESYSTEM_ROOT=$3 +http_proxy=$4 + +TARGET=$TARGET_PATH +[ -z "$TARGET" ] && TARGET=target + +. /usr/local/share/buildinfo/config/buildinfo.config + +TARGET_BASEIMAGE_PATH=$TARGET/versions/host-base-image +mkdir -p $TARGET_BASEIMAGE_PATH + +generate_version_file() +{ + sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg-query -W -f '\${Package}==\${Version}\n'" > $TARGET_BASEIMAGE_PATH/versions-deb-${IMAGE_DISTRO}-${CONFIGURED_ARCH} +} + +if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ]; then + if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then + # qemu arm bin executable for cross-building + sudo mkdir -p $FILESYSTEM_ROOT/usr/bin + sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true + sudo http_proxy=$HTTP_PROXY debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian + else + sudo http_proxy=$HTTP_PROXY debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian + fi + RET=$? + if [ $RET -ne 0 ]; then + exit $RET + fi + + generate_version_file + exit $RET +fi + +ARCH=$(dpkg --print-architecture) +DISTRO=$(grep CODENAME /etc/os-release | cut -d= -f2) +if [ "$ARCH" != "$CONFIGURED_ARCH" ] || [ "$DISTRO" != "$IMAGE_DISTRO" ]; then + "Not support to build different ARCH/DISTRO ${CONFIGURED_ARCH}:${$IMAGE_DISTRO} in ${ARCH}:${DISTRO}." + exit 1 +fi + +BASE_VERSIONS=files/build/versions/host-base-image/versions-deb-${IMAGE_DISTRO} +BASEIMAGE_TARBALLPATH=$TARGET/baseimage +BASEIMAGE_TARBALL=$(realpath -e $TARGET)/baseimage.tgz + +rm -rf $BASEIMAGE_TARBALLPATH $BASEIMAGE_TARBALL + +ARCHIEVES=$BASEIMAGE_TARBALLPATH/var/cache/apt/archives +APTLIST=$BASEIMAGE_TARBALLPATH/var/lib/apt/lists +TARGET_DEBOOTSTRAP=$BASEIMAGE_TARBALLPATH/debootstrap +APTDEBIAN="$APTLIST/deb.debian.org_debian_dists_buster_main_binary-${CONFIGURED_ARCH}_Packages" +DEBPATHS=$TARGET_DEBOOTSTRAP/debpaths +DEBOOTSTRAP_BASE=$TARGET_DEBOOTSTRAP/base +DEBOOTSTRAP_REQUIRED=$TARGET_DEBOOTSTRAP/required +[ -d $BASEIMAGE_TARBALLPATH ] && rm -rf $BASEIMAGE_TARBALLPATH +mkdir -p $ARCHIEVES +mkdir -p $APTLIST +mkdir -p $TARGET_DEBOOTSTRAP +PACKAGES=$(sed -E 's/=(=[^=]*)$/\1/' $BASE_VERSIONS) +URL_ARR=($(apt-get download --print-uris $PACKAGES | cut -d" " -f1 | tr -d "'")) +PACKAGE_ARR=($PACKAGES) +LENGTH=${#PACKAGE_ARR[@]} +for ((i=0;i> $DEBOOTSTRAP_REQUIRED + echo "$packagename /var/cache/apt/archives/$filename" >> $DEBPATHS +done +touch $APTDEBIAN +touch $DEBOOTSTRAP_BASE +(cd $BASEIMAGE_TARBALLPATH && tar -zcf $BASEIMAGE_TARBALL .) + +sudo debootstrap --verbose --variant=minbase --arch $CONFIGURED_ARCH --unpack-tarball=$BASEIMAGE_TARBALL $IMAGE_DISTRO $FILESYSTEM_ROOT +RET=$? +if [ $RET -ne 0 ]; then + exit $RET +fi + +generate_version_file diff --git a/scripts/collect_build_version_files.sh b/scripts/collect_build_version_files.sh new file mode 100755 index 00000000000..485b3b1fd51 --- /dev/null +++ b/scripts/collect_build_version_files.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +RET=$1 +BLDENV=$2 +TARGET_PATH=$3 + +TIMESTAMP=$(date +"%Y%m%d%H%M%S") +. /usr/local/share/buildinfo/config/buildinfo.config + +[ -z "$BLDENV" ] && BLDENV=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) +[ -z "$BLDENV" ] && exit $RET + +[ -z "$TARGET_PATH" ] && TARGET_PATH=./target + +VERSION_BUILD_PATH=$TARGET_PATH/versions/build +VERSION_SLAVE_PATH=$VERSION_BUILD_PATH/build-sonic-slave-${BLDENV} +LOG_VERSION_PATH=$VERSION_BUILD_PATH/log-${TIMESTAMP} + +sudo chmod -R a+rw $BUILDINFO_PATH +collect_version_files $LOG_VERSION_PATH +([ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls $BUILD_VERSION_PATH/)" ]) && cp -rf $BUILD_VERSION_PATH/* $LOG_VERSION_PATH/ +mkdir -p $VERSION_SLAVE_PATH + +scripts/versions_manager.py merge -t $VERSION_SLAVE_PATH -b $LOG_VERSION_PATH -e $POST_VERSION_PATH + +rm -rf $BUILD_VERSION_PATH/* + +exit $RET diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh new file mode 100755 index 00000000000..4523ef84d5e --- /dev/null +++ b/scripts/collect_docker_version_files.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +DOCKER_IMAGE=$1 +TARGET_PATH=$2 + +[ -z "$TARGET_PATH" ] && TARGET_PATH=./target + +DOCKER_IMAGE_NAME=$(echo $DOCKER_IMAGE | cut -d: -f1) +DOCKER_CONTAINER=$DOCKER_IMAGE_NAME +TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME + +[ -d $TARGET_VERSIONS_PATH ] && rm -rf $TARGET_VERSIONS_PATH +mkdir -p $TARGET_VERSIONS_PATH + +export DOCKER_CLI_EXPERIMENTAL=enabled +docker run --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE > /dev/null 2>&1 +docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 +docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/diff-versions $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 +mv $TARGET_VERSIONS_PATH/diff-versions/* $TARGET_VERSIONS_PATH/ +rm -rf $TARGET_VERSIONS_PATH/diff-versions +docker container rm $DOCKER_CONTAINER > /dev/null 2>&1 diff --git a/scripts/collect_host_image_version_files.sh b/scripts/collect_host_image_version_files.sh new file mode 100755 index 00000000000..60d64a9195d --- /dev/null +++ b/scripts/collect_host_image_version_files.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +TARGET=$1 +FILESYSTEM_ROOT=$2 +VERSIONS_PATH=$TARGET/versions/host-image + +mkdir -p $VERSIONS_PATH + +sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo + +cp $FILESYSTEM_ROOT/usr/local/share/buildinfo/diff-versions/* $VERSIONS_PATH/ diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh new file mode 100755 index 00000000000..33af10bed6e --- /dev/null +++ b/scripts/generate_buildinfo_config.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +BUILDINFO_PATH=files/build + +TRUSTED_GPG_PATH=$BUILDINFO_PATH/buildinfo/trusted.gpg.d +BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config +BUILDINFO_CONFIG_TEMPLATE=$BUILDINFO_PATH/templates/buildinfo.config.j2 + +AUTO_GENERATE_CODE_TEXT="Begin Auto-Generated Code" +DOCKERFILE_TEMPLATE="files/build/templates/dockerfile_auto_generate.j2" + +[ -d $TRUSTED_GPG_PATH ] && rm -rf $TRUSTED_GPG_PATH +mkdir -p $TRUSTED_GPG_PATH +mkdir -p $BUILDINFO_PATH/buildinfo/config + +# Generate build info config file +PACKAGE_URL_PREFIX=${PACKAGE_URL_PREFIX} \ +j2 $BUILDINFO_CONFIG_TEMPLATE > $BUILDINFO_CONFIG + + + +# Download trusted gpgs +for url in $(echo $TRUSTED_GPG_URLS | sed 's/[,;]/ /g') +do + wget -q "$url" -P "$TRUSTED_GPG_PATH/" +done diff --git a/scripts/prepare_base_image_buildinfo.sh b/scripts/prepare_base_image_buildinfo.sh new file mode 100755 index 00000000000..230b843c295 --- /dev/null +++ b/scripts/prepare_base_image_buildinfo.sh @@ -0,0 +1,35 @@ +#!/bin/bash + + +ARCH=$1 +DISTRO=$2 +FILESYSTEM_ROOT=$3 + +. /usr/local/share/buildinfo/config/buildinfo.config +VERSION_DEB_PREFERENCE="01-versions-deb" +BUILDINFO_PATH=${FILESYSTEM_ROOT}/usr/local/share/buildinfo +BUILDINFO_INSTALL_PATH=${FILESYSTEM_ROOT}/usr/local/sbin +BUILDINFO_VERSION_PATH=${FILESYSTEM_ROOT}/usr/local/share/buildinfo/versions +BUILDINFO_VERSION_DEB=${BUILDINFO_VERSION_PATH}/${VERSION_DEB_PREFERENCE} +OVERRIDE_VERSION_PATH=files/build/versions/host-image +DIFF_VERSIONS_PATH=$BUILDINFO_PATH/diff-versions + +# Copy build info scripts +mkdir -p $BUILDINFO_PATH +cp -rf files/build/scripts ${BUILDINFO_PATH}/ + +# Copy the build info config +cp -rf files/build/buildinfo/* $BUILDINFO_PATH/ + +# Install the config files +cp -rf $BUILDINFO_PATH/scripts/* "$BUILDINFO_INSTALL_PATH/" +cp $BUILDINFO_PATH/trusted.gpg.d/* "${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/" + +# Generate version lock files +scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -m "$OVERRIDE_VERSION_PATH" -d "$DISTRO" -a "$ARCH" + +if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ]; then + cp -f $BUILDINFO_VERSION_DEB ${FILESYSTEM_ROOT}/etc/apt/preferences.d/ +fi + +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo" diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh new file mode 100755 index 00000000000..6b4f1c8419b --- /dev/null +++ b/scripts/prepare_docker_buildinfo.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +IMAGENAME=$1 +DOCKERFILE=$2 +ARCH=$3 +DOCKERFILE_TARGE=$4 +DISTRO=$5 + +[ -z "$DOCKERFILE_TARGE" ] && DOCKERFILE_TARGE=$DOCKERFILE +DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE") +BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo" +BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions" +BUILDINFO_CONFIG=$BUILDINFO_PATH/scripts/buildinfo.config + +AUTO_GENERATE_CODE_TEXT="Begin Auto-Generated Code" +DOCKERFILE_TEMPLATE="files/build/templates/dockerfile_auto_generate.j2" +[ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH +mkdir -p $BUILDINFO_VERSION_PATH + +# Get the debian distribution from the docker base image +if [ -z "$DISTRO" ]; then + DOCKER_BASE_IMAGE=$(grep "^FROM" $DOCKERFILE | head -n 1 | awk '{print $2}') + DISTRO=$(docker run --rm --entrypoint "" $DOCKER_BASE_IMAGE cat /etc/os-release | grep VERSION_CODENAME | cut -d= -f2) + [ -z "$DISTRO" ] && DISTRO=jessie +fi + +# Add the auto-generate code if it is not added in the target Dockerfile +if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "$AUTO_GENERATE_CODE_TEXT" $DOCKERFILE_TARGE; then + # Insert the docker build script before the RUN command + LINE_NUMBER=$(grep -Fn -m 1 'RUN' $DOCKERFILE | cut -d: -f1) + DOCKERFILE_BEFORE_RUN_SCRIPT=$(generate_code="before_run" j2 $DOCKERFILE_TEMPLATE) + TEMP_FILE=$(mktemp) + awk -v text="${DOCKERFILE_BEFORE_RUN_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE + + # Append the docker build script at the end of the docker file + SET_ENV_PATH=y + [ "$BUILD_SLAVE" == "y" ] && SET_ENV_PATH=n + generate_code="after_run" set_env_path=$SET_ENV_PATH j2 $DOCKERFILE_TEMPLATE >> $TEMP_FILE + + cat $TEMP_FILE > $DOCKERFILE_TARGE + rm -f $TEMP_FILE +fi + +# Copy the build info config +cp -rf files/build/buildinfo/* $BUILDINFO_PATH + +# Copy the docker build info scirpts +cp -rf files/build/scripts "${BUILDINFO_PATH}/" + +# Build the slave running config +if [ "$BUILD_SLAVE" == "y" ]; then + scripts/versions_manager.py generate -t "${BUILDINFO_PATH}/build/versions" -n "build-${IMAGENAME}" -d "$DISTRO" -a "$ARCH" + touch ${BUILDINFO_PATH}/build/versions/versions-deb +fi + +# Generate the version lock files +scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH" + +touch $BUILDINFO_VERSION_PATH/versions-deb diff --git a/scripts/prepare_slave_container_buildinfo.sh b/scripts/prepare_slave_container_buildinfo.sh new file mode 100755 index 00000000000..43c68039d89 --- /dev/null +++ b/scripts/prepare_slave_container_buildinfo.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +SLAVE_DIR=$1 + +BUILDINFO=$SLAVE_DIR/buildinfo +BUILD_VERSIONS_PATH=$SLAVE_DIR/buildinfo/build/versions +VERSION_DEB_PREFERENCE=$BUILD_VERSIONS_PATH/01-versions-deb + +cp -rf $SLAVE_DIR/buildinfo/* /usr/local/share/buildinfo/ + +[ -d /usr/local/share/buildinfo/versions ] && rm -rf /usr/local/share/buildinfo/versions +mkdir -p /usr/local/share/buildinfo/versions +cp -rf $BUILD_VERSIONS_PATH/* /usr/local/share/buildinfo/versions/ + +rm -f /etc/apt/preferences.d/01-versions-deb +[ -f $VERSION_DEB_PREFERENCE ] && cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py new file mode 100755 index 00000000000..0d82955990e --- /dev/null +++ b/scripts/versions_manager.py @@ -0,0 +1,640 @@ +#!/usr/bin/python3 + +import argparse +import glob +import os +import sys + +ALL_DIST = 'all' +ALL_ARCH = 'all' +DEFAULT_MODULE = 'default' +DEFAULT_VERSION_PATH = 'files/build/versions' +VERSION_PREFIX="versions-" +VERSION_DEB_PREFERENCE = '01-versions-deb' +DEFAULT_OVERWRITE_COMPONENTS=['deb', 'py2', 'py3'] +SLAVE_INDIVIDULE_VERSION = False + + +class Component: + ''' + The component consists of mutiple packages + + ctype -- Component Type, such as deb, py2, etc + dist -- Distribution, such as stretch, buster, etc + arch -- Architectrue, such as amd64, arm64, etc + + ''' + def __init__(self, versions, ctype, dist=ALL_DIST, arch=ALL_ARCH): + self.versions = versions + self.ctype = ctype + if not dist: + dist = ALL_DIST + if not arch: + arch = ALL_ARCH + self.dist = dist + self.arch = arch + + @classmethod + def get_versions(cls, version_file): + result = {} + if not os.path.exists(version_file): + return result + with open(version_file) as fp: + for line in fp.readlines(): + offset = line.rfind('==') + if offset > 0: + package = line[:offset].strip() + version = line[offset+2:].strip() + result[package] = version + return result + + def clone(self): + return Component(self.versions.copy(), self.ctype, self.dist, self.arch) + + def merge(self, versions, overwritten=True): + for package in versions: + if overwritten or package not in self.versions: + self.versions[package] = versions[package] + + def subtract(self, versions): + for package in versions: + if package in self.versions and self.versions[package] == versions[package]: + del self.versions[package] + + def dump(self, config=False, priority=999): + result = [] + for package in sorted(self.versions.keys(), key=str.casefold): + if config and self.ctype == 'deb': + lines = 'Package: {0}\nPin: version {1}\nPin-Priority: {2}\n\n'.format(package, self.versions[package], priority) + result.append(lines) + else: + result.append('{0}=={1}'.format(package, self.versions[package])) + return "\n".join(result) + + def dump_to_file(self, version_file, config=False, priority=999): + if len(self.versions) <= 0: + return + with open(version_file, 'w') as f: + f.write(self.dump(config, priority)) + + def dump_to_path(self, file_path, config=False, priority=999): + if len(self.versions) <= 0: + return + if not os.path.exists(file_path): + os.makedirs(file_path) + filename = self.get_filename() + if config and self.ctype == 'deb': + filename = VERSION_DEB_PREFERENCE + file_path = os.path.join(file_path, filename) + self.dump_to_file(file_path, config, priority) + + # Check if the self component can be overwritten by the input component + def check_overwritable(self, component, for_all_dist=False, for_all_arch=False): + if self.ctype != component.ctype: + return False + if self.dist != component.dist and not (for_all_dist and self.dist == ALL_DIST): + return False + if self.arch != component.arch and not (for_all_arch and self.arch == ALL_ARCH): + return False + return True + + # Check if the self component can inherit the package versions from the input component + def check_inheritable(self, component): + if self.ctype != component.ctype: + return False + if self.dist != component.dist and component.dist != ALL_DIST: + return False + if self.arch != component.arch and component.arch != ALL_ARCH: + return False + return True + + ''' + Get the file name + + The file name format: versions-{ctype}-{dist}-{arch} + If {arch} is all, then the file name format: versions-{ctype}-{dist} + if {arch} is all and {dist} is all, then the file name format: versions-{ctype} + ''' + def get_filename(self): + filename = VERSION_PREFIX + self.ctype + dist = self.dist + if self.arch and self.arch != ALL_ARCH: + if not dist: + dist = ALL_DIST + return filename + '-' + dist + '-' + self.arch + if dist and self.dist != ALL_DIST: + filename = filename + '-' + dist + return filename + + def get_order_keys(self): + dist = self.dist + if not dist or dist == ALL_DIST: + dist = '' + arch = self.arch + if not arch or arch == ALL_ARCH: + arch = '' + return (self.ctype, dist, arch) + + def clean_info(self, clean_dist=True, clean_arch=True): + if clean_dist and self.ctype != 'deb': + self.dist = ALL_DIST + if clean_arch: + self.arch = ALL_ARCH + + +class VersionModule: + ''' + The version module represents a build target, such as docker image, host image, consists of multiple components. + + name -- The name of the image, such as sonic-slave-buster, docker-lldp, etc + ''' + def __init__(self, name=None, components=None): + self.name = name + self.components = components + + # Overwrite the docker/host image/base image versions + def overwrite(self, module, for_all_dist=False, for_all_arch=False): + # Overwrite from generic one to detail one + # For examples: versions-deb overwrtten by versions-deb-buster, and versions-deb-buster overwritten by versions-deb-buster-amd64 + components = sorted(module.components, key = lambda x : x.get_order_keys()) + for merge_component in components: + merged = False + for component in self.components: + if component.check_overwritable(merge_component, for_all_dist=for_all_dist, for_all_arch=for_all_arch): + component.merge(merge_component.versions, True) + merged = True + if not merged: + tmp_component = merge_component.clone() + tmp_component.clean_info(clean_dist=for_all_dist, clean_arch=for_all_arch) + self.components.append(tmp_component) + self.adjust() + + def get_config_module(self, default_module, dist, arch): + if self.is_individule_version(): + return self + module = default_module.clone() + module.overwrite(self) + config_components = [] + ctype_components = module._get_components_per_ctypes() + default_ctype_components = default_module._get_components_per_ctypes() + for ctype in default_ctype_components: + if ctype not in ctype_components: + ctype_components[ctype] = [] + for components in ctype_components.values(): + config_component = self._get_config_for_ctype(components, dist, arch) + config_components.append(config_component) + config_module = VersionModule(self.name, config_components) + return config_module + + def _get_config_for_ctype(self, components, dist, arch): + result = Component({}, components[0].ctype, dist, arch) + for component in sorted(components, key = lambda x : x.get_order_keys()): + if result.check_inheritable(component): + result.merge(component.versions, True) + return result + + def subtract(self, default_module): + module = self.clone() + result = [] + ctype_components = module._get_components_per_ctypes() + for ctype in ctype_components: + components = ctype_components[ctype] + components = sorted(components, key = lambda x : x.get_order_keys()) + for i in range(0, len(components)): + component = components[i] + base_module = VersionModule(self.name, components[0:i]) + config_module = base_module.get_config_module(default_module, component.dist, component.arch) + config_components = config_module._get_components_by_ctype(ctype) + if len(config_components) > 0: + config_component = config_components[0] + component.subtract(config_component.versions) + if len(component.versions): + result.append(component) + self.components = result + + def adjust(self): + result_components = [] + ctype_components = self._get_components_per_ctypes() + for components in ctype_components.values(): + result_components += self._adjust_components_for_ctype(components) + self.components = result_components + + def _get_components_by_ctype(self, ctype): + components = [] + for component in self.components: + if component.ctype == ctype: + components.append(component) + return components + + def _adjust_components_for_ctype(self, components): + components = sorted(components, key = lambda x : x.get_order_keys()) + result = [] + for i in range(0, len(components)): + component = components[i] + inheritable_component = Component({}, component.ctype) + for j in range(0, i): + base_component = components[j] + if component.check_inheritable(base_component): + inheritable_component.merge(base_component.versions, True) + component.subtract(inheritable_component.versions) + if len(component.versions) > 0: + result.append(component) + return result + + def _get_components_per_ctypes(self): + result = {} + for component in self.components: + components = result.get(component.ctype, []) + components.append(component) + result[component.ctype] = components + return result + + def load(self, image_path, filter_ctype=None, filter_dist=None, filter_arch=None): + version_file_pattern = os.path.join(image_path, VERSION_PREFIX) + '*' + file_paths = glob.glob(version_file_pattern) + components = [] + self.name = os.path.basename(image_path) + self.components = components + for file_path in file_paths: + filename = os.path.basename(file_path) + items = filename.split('-') + if len(items) < 2: + continue + ctype = items[1] + if filter_ctype and filter_ctype != ctype: + continue + dist = '' + arch = '' + if len(items) > 2: + dist = items[2] + if filter_dist and dist and filter_dist != dist: + continue + if len(items) > 3: + arch = items[3] + if filter_arch and arch and filter_arch != arch: + continue + versions = Component.get_versions(file_path) + component = Component(versions, ctype, dist, arch) + components.append(component) + + def load_from_target(self, image_path): + self.load(image_path) + + def dump(self, module_path, config=False, priority=999): + version_file_pattern = os.path.join(module_path, VERSION_PREFIX + '*') + for filename in glob.glob(version_file_pattern): + os.remove(filename) + for component in self.components: + component.dump_to_path(module_path, config, priority) + + def filter(self, ctypes=[]): + if 'all' in ctypes: + return self + components = [] + for component in self.components: + if component.ctype in ctypes: + components.append(component) + self.components = components + + def clean_info(self, clean_dist=True, clean_arch=True): + for component in self.components: + component.clean_info(clean_dist=clean_dist, clean_arch=clean_arch) + + def clone(self, ctypes=None, exclude_ctypes=None): + components = [] + for component in self.components: + if exclude_ctypes and component.ctype in exclude_ctypes: + continue + if ctypes and component.ctype not in ctypes: + continue + components.append(component.clone()) + return VersionModule(self.name, components) + + def is_slave_module(self): + return self.name.startswith('sonic-slave-') + + # Do not inherit the version from the default module + def is_individule_version(self): + return self.is_slave_module() and SLAVE_INDIVIDULE_VERSION + + @classmethod + def get_module_path_by_name(cls, source_path, module_name): + common_modules = ['default', 'host-image', 'host-base-image'] + if module_name in common_modules: + return os.path.join(source_path, 'files/build/versions', module_name) + if module_name.startswith('build-sonic-slave-'): + return os.path.join(source_path, 'files/build/versions/build', module_name) + return os.path.join(source_path, 'files/build/versions/dockers', module_name) + +class VersionBuild: + ''' + The VersionBuild consists of multiple version modules. + + ''' + def __init__(self, target_path="./target", source_path='.'): + self.target_path = target_path + self.source_path = source_path + self.modules = {} + + def load_from_target(self): + dockers_path = os.path.join(self.target_path, 'versions/dockers') + build_path = os.path.join(self.target_path, 'versions/build') + modules = {} + self.modules = modules + file_paths = glob.glob(dockers_path + '/*') + file_paths += glob.glob(build_path + '/build-*') + file_paths.append(os.path.join(self.target_path, 'versions/host-image')) + file_paths.append(os.path.join(self.target_path, 'versions/host-base-image')) + for file_path in file_paths: + if not os.path.isdir(file_path): + continue + module = VersionModule() + module.load_from_target(file_path) + modules[module.name] = module + self._merge_dgb_modules() + + def load_from_source(self): + # Load default versions and host image versions + versions_path = os.path.join(self.source_path, 'files/build/versions') + dockers_path = os.path.join(versions_path, "dockers") + build_path = os.path.join(versions_path, "build") + paths = [os.path.join(versions_path, 'default')] + paths += glob.glob(versions_path + '/host-*') + paths += glob.glob(dockers_path + '/*') + paths += glob.glob(build_path + '/*') + modules = {} + self.modules = modules + for image_path in paths: + module = VersionModule() + module.load(image_path) + modules[module.name] = module + + def overwrite(self, build, for_all_dist=False, for_all_arch=False): + for target_module in build.modules.values(): + module = self.modules.get(target_module.name, None) + tmp_module = target_module.clone() + tmp_module.clean_info(for_all_dist, for_all_arch) + if module: + module.overwrite(tmp_module, for_all_dist=for_all_dist, for_all_arch=for_all_arch) + else: + self.modules[target_module.name] = tmp_module + + def dump(self): + for module in self.modules.values(): + module_path = self.get_module_path(module) + module.dump(module_path) + + def subtract(self, default_module): + for module in self.modules.values(): + if module.name == DEFAULT_MODULE: + continue + if module.name == 'host-base-image': + continue + if module.is_individule_version(): + continue + module.subtract(default_module) + + def freeze(self, rebuild=False, for_all_dist=False, for_all_arch=False, ctypes=['all']): + if rebuild: + self.load_from_target() + self.filter(ctypes=ctypes) + default_module = self.get_default_module() + self._clean_component_info() + self.subtract(default_module) + self.modules[DEFAULT_MODULE] = default_module + self.dump() + return + self.load_from_source() + default_module = self.modules.get(DEFAULT_MODULE, None) + target_build = VersionBuild(self.target_path, self.source_path) + target_build.load_from_target() + target_build.filter(ctypes=ctypes) + if not default_module: + raise Exception("The default versions does not exist") + for module in target_build.modules.values(): + if module.is_individule_version(): + continue + tmp_module = module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) + default_module.overwrite(tmp_module, for_all_dist=True, for_all_arch=True) + target_build.subtract(default_module) + self.overwrite(target_build, for_all_dist=for_all_dist, for_all_arch=for_all_arch) + self.dump() + + def filter(self, ctypes=[]): + for module in self.modules.values(): + module.filter(ctypes=ctypes) + + def get_default_module(self): + if DEFAULT_MODULE in self.modules: + return self.modules[DEFAULT_MODULE] + ctypes = self.get_component_types() + dists = self.get_dists() + components = [] + for ctype in ctypes: + if ctype == 'deb': + for dist in dists: + versions = self._get_versions(ctype, dist) + common_versions = self._get_common_versions(versions) + component = Component(common_versions, ctype, dist) + components.append(component) + else: + versions = self._get_versions(ctype) + common_versions = self._get_common_versions(versions) + component = Component(common_versions, ctype) + components.append(component) + return VersionModule(DEFAULT_MODULE, components) + + def get_docker_version_modules(self): + modules = {} + for module_name in self.modules: + if module_name.startswith('sonic-slave-'): + continue + if module_name.startswith('build-sonic-slave-'): + continue + if module_name == DEFAULT_MODULE: + continue + if module_name == 'host-image' or module_name == 'host-base-image': + continue + module = self.modules[module_name] + modules[module_name] = module + return modules + + def get_components(self): + components = [] + for module_name in self.modules: + module = self.modules[module_name] + for component in module.components: + components.append(component) + return components + + def get_component_types(self): + ctypes = [] + for module_name in self.modules: + module = self.modules[module_name] + for component in module.components: + if component.ctype not in ctypes: + ctypes.append(component.ctype) + return ctypes + + def get_dists(self): + dists = [] + components = self.get_components() + for component in components: + if component.dist not in dists: + dists.append(component.dist) + return dists + + def get_archs(self): + archs = [] + components = self.get_components() + for component in components: + if component.arch not in archs: + archs.append(component.arch) + return archs + + def get_module_path(self, module): + return self.get_module_path_by_name(module.name) + + def get_module_path_by_name(self, module_name): + return VersionModule.get_module_path_by_name(self.source_path, module_name) + + def _merge_dgb_modules(self): + dbg_modules = [] + for module_name in self.modules: + if not module_name.endswith('-dbg'): + continue + dbg_modules.append(module_name) + base_module_name = module_name[:-4] + if base_module_name not in self.modules: + raise Exception('The Module {0} not found'.format(base_module_name)) + base_module = self.modules[base_module_name] + dbg_module = self.modules[module_name] + base_module.overwrite(dbg_module) + for module_name in dbg_modules: + del self.modules[module_name] + + def _clean_component_info(self, clean_dist=True, clean_arch=True): + for module in self.modules.values(): + module.clean_info(clean_dist, clean_arch) + + def _get_versions(self, ctype, dist=None, arch=None): + versions = {} + modules = self.get_docker_version_modules() + for module_name in self.modules: + if module_name not in modules: + temp_module = self.modules[module_name].clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) + modules[module_name] = temp_module + for module in modules.values(): + for component in module.components: + if ctype != component.ctype: + continue + if dist and dist != component.dist: + continue + if arch and arch != component.arch: + continue + for package in component.versions: + version = component.versions[package] + package_versions = versions.get(package, []) + if version not in package_versions: + package_versions.append(version) + versions[package] = package_versions + return versions + + def _get_common_versions(self, versions): + common_versions = {} + for package in versions: + package_versions = versions[package] + if len(package_versions) == 1: + common_versions[package] = package_versions[0] + return common_versions + + +class VersionManagerCommands: + def __init__(self): + usage = 'version_manager.py []\n\n' + usage = usage + 'The most commonly used commands are:\n' + usage = usage + ' freeze Freeze the version files\n' + usage = usage + ' generate Generate the version files\n' + usage = usage + ' merge Merge the version files' + parser = argparse.ArgumentParser(description='Version manager', usage=usage) + parser.add_argument('command', help='Subcommand to run') + args = parser.parse_args(sys.argv[1:2]) + if not hasattr(self, args.command): + print('Unrecognized command: {0}'.format(args.command)) + parser.print_help() + exit(1) + getattr(self, args.command)() + + def freeze(self): + parser = argparse.ArgumentParser(description = 'Freeze the version files') + parser.add_argument('-t', '--target_path', default='./target', help='target path') + parser.add_argument('-s', '--source_path', default='.', help='source path') + + # store_true which implies default=False + parser.add_argument('-r', '--rebuild', action='store_true', help='rebuild all versions') + parser.add_argument('-d', '--for_all_dist', action='store_true', help='apply the versions for all distributions') + parser.add_argument('-a', '--for_all_arch', action='store_true', help='apply the versions for all architectures') + parser.add_argument('-c', '--ctypes', default='all', help='component types to freeze') + args = parser.parse_args(sys.argv[2:]) + ctypes = args.ctypes.split(',') + if len(ctypes) == 0: + ctypes = ['all'] + build = VersionBuild(target_path=args.target_path, source_path=args.source_path) + build.freeze(rebuild=args.rebuild, for_all_dist=args.for_all_dist, for_all_arch=args.for_all_arch, ctypes=ctypes) + + def merge(self): + parser = argparse.ArgumentParser(description = 'Merge the version files') + parser.add_argument('-t', '--target_path', required=True, help='target path to save the merged version files') + parser.add_argument('-m', '--module_path', default=None, help='merge path, use the target path if not specified') + parser.add_argument('-b', '--base_path', required=True, help='base path, merge to the module path') + parser.add_argument('-e', '--exclude_module_path', default=None, help='exclude module path') + args = parser.parse_args(sys.argv[2:]) + module_path = args.module_path + if not module_path: + module_path = args.target_path + if not os.path.exists(module_path): + print('The module path {0} does not exist'.format(module_path)) + if not os.path.exists(args.target_path): + os.makedirs(args.target_path) + module = VersionModule() + module.load(module_path) + base_module = VersionModule() + base_module.load(args.base_path) + module.overwrite(base_module) + if args.exclude_module_path: + exclude_module = VersionModule() + exclude_module.load(args.exclude_module_path) + module.subtract(exclude_module) + module.dump(args.target_path) + + def generate(self): + script_path = os.path.dirname(sys.argv[0]) + root_path = os.path.dirname(script_path) + + parser = argparse.ArgumentParser(description = 'Generate the version files') + parser.add_argument('-t', '--target_path', required=True, help='target path to generate the version lock files') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-n', '--module_name', help="module name, such as docker-lldp, sonic-slave-buster, etc") + group.add_argument('-m', '--module_path', help="module apth, such as files/docker/versions/dockers/docker-lldp, files/docker/versions/dockers/sonic-slave-buster, etc") + parser.add_argument('-s', '--source_path', default='.', help='source path') + parser.add_argument('-d', '--distribution', required=True, help="distribution") + parser.add_argument('-a', '--architecture', required=True, help="architecture") + parser.add_argument('-p', '--priority', default=999, help="priority of the debian apt preference") + + args = parser.parse_args(sys.argv[2:]) + module_path = args.module_path + if not module_path: + module_path = VersionModule.get_module_path_by_name(args.source_path, args.module_name) + if not os.path.exists(args.target_path): + os.makedirs(args.target_path) + module = VersionModule() + module.load(module_path, filter_dist=args.distribution, filter_arch=args.architecture) + default_module_path = VersionModule.get_module_path_by_name(args.source_path, DEFAULT_MODULE) + default_module = VersionModule() + default_module.load(default_module_path, filter_dist=args.distribution, filter_arch=args.architecture) + config = module.get_config_module(default_module, args.distribution, args.architecture) + config.clean_info() + config.dump(args.target_path, config=True, priority=args.priority) + +if __name__ == "__main__": + VersionManagerCommands() diff --git a/slave.mk b/slave.mk index eb8e9d7765c..29e153e2cd4 100644 --- a/slave.mk +++ b/slave.mk @@ -256,6 +256,9 @@ else $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH)) endif +# Overwrite the buildinfo in slave container +$(shell sudo scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR)) + include Makefile.cache ifeq ($(SONIC_USE_DOCKER_BUILDKIT),y) @@ -636,6 +639,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g $(HEADER) # Apply series of patches if exist if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && QUILT_PATCHES=../$(notdir $($*.gz_PATH)).patch quilt push -a; popd; fi + # Prepare docker build info + scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) $(TARGET_DOCKERFILE)/Dockerfile.buildinfo docker info $(LOG) docker build --squash --no-cache \ --build-arg http_proxy=$(HTTP_PROXY) \ @@ -645,7 +650,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_SIMPLE_DOCKER_IMAGES)) : $(TARGET_PATH)/%.g --build-arg guid=$(GUID) \ --build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \ --label Tag=$(SONIC_IMAGE_VERSION) \ + -f $(TARGET_DOCKERFILE)/Dockerfile.buildinfo \ -t $* $($*.gz_PATH) $(LOG) + scripts/collect_docker_version_files.sh $* $(TARGET_PATH) docker save $* | gzip -c > $@ # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -719,6 +726,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform $(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_whls=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_PYTHON_WHEELS)))\n" | awk '!a[$$0]++')) $(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_PACKAGES)))\n" | awk '!a[$$0]++')) j2 $($*.gz_PATH)/Dockerfile.j2 > $($*.gz_PATH)/Dockerfile + # Prepare docker build info + PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ + scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile $(CONFIGURED_ARCH) docker info $(LOG) docker build --squash --no-cache \ --build-arg http_proxy=$(HTTP_PROXY) \ @@ -731,6 +743,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform --build-arg frr_user_gid=$(FRR_USER_GID) \ --label Tag=$(SONIC_IMAGE_VERSION) \ -t $* $($*.gz_PATH) $(LOG) + scripts/collect_docker_version_files.sh $* $(TARGET_PATH) docker save $* | gzip -c > $@ # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -763,6 +776,11 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG $(eval export $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs=$(shell printf "$(subst $(SPACE),\n,$(call expand,$($*.gz_DBG_IMAGE_PACKAGES)))\n" | awk '!a[$$0]++')) ./build_debug_docker_j2.sh $* $(subst -,_,$(notdir $($*.gz_PATH)))_dbg_debs $(subst -,_,$(notdir $($*.gz_PATH)))_image_dbgs > $($*.gz_PATH)/Dockerfile-dbg.j2 j2 $($*.gz_PATH)/Dockerfile-dbg.j2 > $($*.gz_PATH)/Dockerfile-dbg + # Prepare docker build info + PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ + scripts/prepare_docker_buildinfo.sh $* $($*.gz_PATH)/Dockerfile-dbg $(CONFIGURED_ARCH) docker info $(LOG) docker build \ $(if $($*.gz_DBG_DEPENDS), --squash --no-cache, --no-cache) \ @@ -772,6 +790,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_DBG_IMAGES)) : $(TARGET_PATH)/%-$(DBG_IMAG --label Tag=$(SONIC_IMAGE_VERSION) \ --file $($*.gz_PATH)/Dockerfile-dbg \ -t $*-dbg $($*.gz_PATH) $(LOG) + scripts/collect_docker_version_files.sh $*-dbg $(TARGET_PATH) docker save $*-dbg | gzip -c > $@ # Clean up if [ -f $($*.gz_PATH).patch/series ]; then pushd $($*.gz_PATH) && quilt pop -a -f; [ -d .pc ] && rm -rf .pc; popd; fi @@ -968,6 +987,10 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ USERNAME="$(USERNAME)" \ PASSWORD="$(PASSWORD)" \ IMAGE_TYPE=$($*_IMAGE_TYPE) \ + TARGET_PATH=$(TARGET_PATH) \ + SONIC_ENFORCE_VERSIONS=$(SONIC_ENFORCE_VERSIONS) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ + PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ ./build_debian.sh $(LOG) USERNAME="$(USERNAME)" \ diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index b5d4ac901bd..e9bb9b21dd1 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -16,7 +16,9 @@ RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ bust echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list + echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list {%- if CONFIGURED_ARCH == "armhf" %} RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \ @@ -25,7 +27,9 @@ RUN echo "deb [arch=armhf] http://deb.debian.org/debian buster main contrib non- echo "deb-src [arch=armhf] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list + echo 'deb [arch=armhf] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \ + echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list {%- elif CONFIGURED_ARCH == "arm64" %} RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list && \ echo "deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free" >> /etc/apt/sources.list && \ @@ -33,9 +37,12 @@ RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non- echo "deb-src [arch=arm64] http://deb.debian.org/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list + echo 'deb [arch=arm64] http://ftp.debian.org/debian buster-backports main' >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list {%- endif %} + ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index 75230d29164..3827face126 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -14,7 +14,10 @@ RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stre echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian stretch-backports main" >> /etc/apt/sources.list + echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian stretch-backports main" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=amd64] http://packages.microsoft.com/debian/9/prod stretch main" >> /etc/apt/sources.list {%- if CONFIGURED_ARCH == "armhf" %} RUN echo "deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \ @@ -23,7 +26,9 @@ RUN echo "deb [arch=armhf] http://deb.debian.org/debian stretch main contrib non echo "deb-src [arch=armhf] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=armhf] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list + echo 'deb [arch=armhf] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \ + echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list {%- elif CONFIGURED_ARCH == "arm64" %} RUN echo "deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" > /etc/apt/sources.list && \ echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ @@ -31,9 +36,12 @@ RUN echo "deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non echo "deb-src [arch=arm64] http://deb.debian.org/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ echo "deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free" >> /etc/apt/sources.list && \ - echo 'deb [arch=arm64] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list + echo 'deb [arch=arm64] http://ftp.debian.org/debian stretch-backports main' >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free" >> /etc/apt/sources.list && \ + echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list {%- endif %} + ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive From 0676da0be7b8f5ae2a6e2868e3edb778e126f407 Mon Sep 17 00:00:00 2001 From: xumia Date: Wed, 18 Nov 2020 05:58:33 +0000 Subject: [PATCH 02/18] Remove j2 files --- files/build/scripts/apt-get | 47 ++++++++++--------- .../buildinfo_base.sh} | 4 +- files/build/scripts/pip | 4 +- files/build/scripts/post_run_buildinfo | 2 +- files/build/scripts/pre_run_buildinfo | 2 +- files/build/scripts/wget | 2 +- .../templates/dockerfile_auto_generate.j2 | 13 ----- scripts/build_host_base_image.sh | 2 +- scripts/collect_build_version_files.sh | 2 +- scripts/generate_buildinfo_config.sh | 10 +--- scripts/prepare_base_image_buildinfo.sh | 2 +- scripts/prepare_docker_buildinfo.sh | 24 ++++++---- slave.mk | 8 ++++ 13 files changed, 60 insertions(+), 62 deletions(-) rename files/build/{templates/buildinfo.config.j2 => scripts/buildinfo_base.sh} (90%) mode change 100644 => 100755 delete mode 100644 files/build/templates/dockerfile_auto_generate.j2 diff --git a/files/build/scripts/apt-get b/files/build/scripts/apt-get index 4fe2659be19..99b266f6f0f 100755 --- a/files/build/scripts/apt-get +++ b/files/build/scripts/apt-get @@ -1,35 +1,38 @@ #!/bin/bash INSTALL= +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh VERSION_FILE="/usr/local/share/buildinfo/base-versions/versions-deb" -for para in $@ -do - if [[ "$para" != -* ]]; then - continue - fi - if [ ! -z "$INSTALL" ]; then - if [[ "$para" == *=* ]]; then +if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then + for para in $@ + do + if [[ "$para" != -* ]]; then continue - package=$(echo "$para" | cut -d= -f1) - version=$(echo "$para" | cut -d= -f2) - if grep "^${package}=" $VERSION_FILE; then - if ! grep "^${package}==${version}$" $VERSION_FILE; then - EXPECTED_VERSION=$(grep "^${package}=" $VERSION_FILE | cut -d= -f3) - echo "${package}=${version}, expected version is ${EXPECTED_VERSION}" + fi + if [ ! -z "$INSTALL" ]; then + if [[ "$para" == *=* ]]; then + continue + package=$(echo "$para" | cut -d= -f1) + version=$(echo "$para" | cut -d= -f2) + if grep "^${package}=" $VERSION_FILE; then + if ! grep "^${package}==${version}$" $VERSION_FILE; then + EXPECTED_VERSION=$(grep "^${package}=" $VERSION_FILE | cut -d= -f3) + echo "${package}=${version}, expected version is ${EXPECTED_VERSION}" + exit 1 + fi + fi + else + if ! grep "^${package}=" $VERSION_FILE; then + echo "The version of the package ${package} is not specified." exit 1 fi fi - else - if ! grep "^${package}=" $VERSION_FILE; then - echo "The version of the package ${package} is not specified." - exit 1 - fi + elif [[ "$para" == "install" ]]; then + INSTALL=y fi - elif [[ "$para" == "install" ]]; then - INSTALL=y - fi -done + done +fi /usr/bin/apt-get $@ diff --git a/files/build/templates/buildinfo.config.j2 b/files/build/scripts/buildinfo_base.sh old mode 100644 new mode 100755 similarity index 90% rename from files/build/templates/buildinfo.config.j2 rename to files/build/scripts/buildinfo_base.sh index ee0d3d64a37..b9c3f688e69 --- a/files/build/templates/buildinfo.config.j2 +++ b/files/build/scripts/buildinfo_base.sh @@ -10,8 +10,8 @@ BUILD_VERSION_PATH=$BUILDINFO_PATH/build-versions POST_VERSION_PATH=$BUILDINFO_PATH/post-versions VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb -PACKAGE_URL_PREFIX={{ PACKAGE_URL_PREFIX }} -SONIC_VERSION_CONTROL_COMPONENTS={{ SONIC_VERSION_CONTROL_COMPONENTS }} +. $BUILDINFO_PATH/config/buildinfo.config + URL_PREFIX=$(echo "${PACKAGE_URL_PREFIX}" | sed -E "s#(//[^/]*/).*#\1#") log_err() diff --git a/files/build/scripts/pip b/files/build/scripts/pip index cbc004650bc..cb9c4e335d0 100755 --- a/files/build/scripts/pip +++ b/files/build/scripts/pip @@ -1,6 +1,6 @@ #!/bin/bash - -. /usr/local/share/buildinfo/config/buildinfo.config + +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh if [ -z "$PIP_COMMAND" ]; then diff --git a/files/build/scripts/post_run_buildinfo b/files/build/scripts/post_run_buildinfo index c9de5341cc9..1b2cdbe2124 100755 --- a/files/build/scripts/post_run_buildinfo +++ b/files/build/scripts/post_run_buildinfo @@ -1,6 +1,6 @@ #!/bin/bash -. /usr/local/share/buildinfo/config/buildinfo.config +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh [ -d $POST_VERSION_PATH ] && rm -rf $POST_VERSION_PATH diff --git a/files/build/scripts/pre_run_buildinfo b/files/build/scripts/pre_run_buildinfo index 9299b37802f..961277f7545 100755 --- a/files/build/scripts/pre_run_buildinfo +++ b/files/build/scripts/pre_run_buildinfo @@ -1,6 +1,6 @@ #!/bin/bash -. /usr/local/share/buildinfo/config/buildinfo.config +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh [ -d $DIFF_VERSION_PATH ] && rm -rf $DIFF_VERSION_PATH mkdir -p $DIFF_VERSION_PATH diff --git a/files/build/scripts/wget b/files/build/scripts/wget index b78806176d6..40675992c84 100755 --- a/files/build/scripts/wget +++ b/files/build/scripts/wget @@ -1,6 +1,6 @@ #!/bin/bash -. /usr/local/share/buildinfo/config/buildinfo.config +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh COMPONENT=web VERSION_FILE=$VERSION_PATH/versions-web BUILD_VERSION_FILE=$BUILD_VERSION_PATH/versions-web diff --git a/files/build/templates/dockerfile_auto_generate.j2 b/files/build/templates/dockerfile_auto_generate.j2 deleted file mode 100644 index 8fc473583d8..00000000000 --- a/files/build/templates/dockerfile_auto_generate.j2 +++ /dev/null @@ -1,13 +0,0 @@ -{% if generate_code == 'before_run' %} -# ---- Begin Auto-Generated Code ---- -COPY ["buildinfo", "/usr/local/share/buildinfo"] -ENV OLDPATH=$PATH -ENV PATH="/usr/local/share/buildinfo/scripts:$PATH" -RUN pre_run_buildinfo -# --- End Auto-Generated Code ---- -{% elif generate_code == 'after_run' %} -# ---- Begin Auto-Generated Code ---- -RUN post_run_buildinfo -{% if env("set_env_path", default='y') == 'y' %}ENV PATH=$OLDPATH{% endif %} -# ---- End Auto-Generated Code ---- -{% endif %} diff --git a/scripts/build_host_base_image.sh b/scripts/build_host_base_image.sh index dca4d173a9c..e3ec1dcf669 100755 --- a/scripts/build_host_base_image.sh +++ b/scripts/build_host_base_image.sh @@ -9,7 +9,7 @@ http_proxy=$4 TARGET=$TARGET_PATH [ -z "$TARGET" ] && TARGET=target -. /usr/local/share/buildinfo/config/buildinfo.config +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh TARGET_BASEIMAGE_PATH=$TARGET/versions/host-base-image mkdir -p $TARGET_BASEIMAGE_PATH diff --git a/scripts/collect_build_version_files.sh b/scripts/collect_build_version_files.sh index 485b3b1fd51..b650e421996 100755 --- a/scripts/collect_build_version_files.sh +++ b/scripts/collect_build_version_files.sh @@ -5,7 +5,7 @@ BLDENV=$2 TARGET_PATH=$3 TIMESTAMP=$(date +"%Y%m%d%H%M%S") -. /usr/local/share/buildinfo/config/buildinfo.config +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh [ -z "$BLDENV" ] && BLDENV=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) [ -z "$BLDENV" ] && exit $RET diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh index 33af10bed6e..5e30addf5de 100755 --- a/scripts/generate_buildinfo_config.sh +++ b/scripts/generate_buildinfo_config.sh @@ -4,19 +4,13 @@ BUILDINFO_PATH=files/build TRUSTED_GPG_PATH=$BUILDINFO_PATH/buildinfo/trusted.gpg.d BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config -BUILDINFO_CONFIG_TEMPLATE=$BUILDINFO_PATH/templates/buildinfo.config.j2 - -AUTO_GENERATE_CODE_TEXT="Begin Auto-Generated Code" -DOCKERFILE_TEMPLATE="files/build/templates/dockerfile_auto_generate.j2" [ -d $TRUSTED_GPG_PATH ] && rm -rf $TRUSTED_GPG_PATH mkdir -p $TRUSTED_GPG_PATH mkdir -p $BUILDINFO_PATH/buildinfo/config -# Generate build info config file -PACKAGE_URL_PREFIX=${PACKAGE_URL_PREFIX} \ -j2 $BUILDINFO_CONFIG_TEMPLATE > $BUILDINFO_CONFIG - +echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG +echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG # Download trusted gpgs diff --git a/scripts/prepare_base_image_buildinfo.sh b/scripts/prepare_base_image_buildinfo.sh index 230b843c295..588827c7f86 100755 --- a/scripts/prepare_base_image_buildinfo.sh +++ b/scripts/prepare_base_image_buildinfo.sh @@ -5,7 +5,7 @@ ARCH=$1 DISTRO=$2 FILESYSTEM_ROOT=$3 -. /usr/local/share/buildinfo/config/buildinfo.config +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh VERSION_DEB_PREFERENCE="01-versions-deb" BUILDINFO_PATH=${FILESYSTEM_ROOT}/usr/local/share/buildinfo BUILDINFO_INSTALL_PATH=${FILESYSTEM_ROOT}/usr/local/sbin diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 6b4f1c8419b..1a8087cfc40 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -10,10 +10,7 @@ DISTRO=$5 DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE") BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo" BUILDINFO_VERSION_PATH="${BUILDINFO_PATH}/versions" -BUILDINFO_CONFIG=$BUILDINFO_PATH/scripts/buildinfo.config -AUTO_GENERATE_CODE_TEXT="Begin Auto-Generated Code" -DOCKERFILE_TEMPLATE="files/build/templates/dockerfile_auto_generate.j2" [ -d $BUILDINFO_PATH ] && rm -rf $BUILDINFO_PATH mkdir -p $BUILDINFO_VERSION_PATH @@ -24,18 +21,27 @@ if [ -z "$DISTRO" ]; then [ -z "$DISTRO" ] && DISTRO=jessie fi +DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo +COPY ["buildinfo", "/usr/local/share/buildinfo"] +ENV OLDPATH=$PATH +ENV PATH="/usr/local/share/buildinfo/scripts:$PATH" +RUN pre_run_buildinfo' + +DOCKERFILE_POST_SCRIPT="RUN post_run_buildinfo" +[ "$BUILD_SLAVE" != "y" ] && DOCKERFILE_POST_SCRIPT="$DOCKERFILE_POST_SCRIPT +ENV PATH=\$OLDPATH" + + # Add the auto-generate code if it is not added in the target Dockerfile -if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "$AUTO_GENERATE_CODE_TEXT" $DOCKERFILE_TARGE; then +if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then # Insert the docker build script before the RUN command LINE_NUMBER=$(grep -Fn -m 1 'RUN' $DOCKERFILE | cut -d: -f1) - DOCKERFILE_BEFORE_RUN_SCRIPT=$(generate_code="before_run" j2 $DOCKERFILE_TEMPLATE) TEMP_FILE=$(mktemp) - awk -v text="${DOCKERFILE_BEFORE_RUN_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE + awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE # Append the docker build script at the end of the docker file - SET_ENV_PATH=y - [ "$BUILD_SLAVE" == "y" ] && SET_ENV_PATH=n - generate_code="after_run" set_env_path=$SET_ENV_PATH j2 $DOCKERFILE_TEMPLATE >> $TEMP_FILE + echo "RUN post_run_buildinfo" >> $TEMP_FILE + [ "$BUILD_SLAVE" != "y" ] && echo "ENV PATH=\$OLDPATH" >> $TEMP_FILE cat $TEMP_FILE > $DOCKERFILE_TARGE rm -f $TEMP_FILE diff --git a/slave.mk b/slave.mk index 29e153e2cd4..bf1bd04508a 100644 --- a/slave.mk +++ b/slave.mk @@ -100,6 +100,14 @@ list : include $(RULES_PATH)/config -include $(RULES_PATH)/config.user + +############################################################################### +## Version control relatived exports +############################################################################### +export PACKAGE_URL_PREFIX +export TRUSTED_GPG_URLS +export SONIC_VERSION_CONTROL_COMPONENTS + ifeq ($(SONIC_ENABLE_PFCWD_ON_START),y) ENABLE_PFCWD_ON_START = y endif From 707892275f3c8b47606a50d99da98984b84d307b Mon Sep 17 00:00:00 2001 From: xumia Date: Wed, 18 Nov 2020 11:26:06 +0000 Subject: [PATCH 03/18] Fix bug --- dockers/docker-base-buster/Dockerfile.j2 | 3 --- dockers/docker-base-buster/sources.list | 2 ++ dockers/docker-base-buster/sources.list.arm64 | 2 ++ dockers/docker-base-buster/sources.list.armhf | 2 ++ files/build/scripts/apt-get | 16 +++++----------- files/build/scripts/buildinfo_base.sh | 4 ++-- files/build/scripts/wget | 7 +++---- rules/config | 6 +++--- scripts/prepare_docker_buildinfo.sh | 7 ++----- scripts/versions_manager.py | 15 +++++++++------ 10 files changed, 30 insertions(+), 34 deletions(-) diff --git a/dockers/docker-base-buster/Dockerfile.j2 b/dockers/docker-base-buster/Dockerfile.j2 index c6d8329b5b8..bf1d8197bf4 100644 --- a/dockers/docker-base-buster/Dockerfile.j2 +++ b/dockers/docker-base-buster/Dockerfile.j2 @@ -7,9 +7,6 @@ FROM multiarch/debian-debootstrap:arm64-buster FROM debian:buster {% endif %} -RUN echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list - # Clean documentation in FROM image RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true diff --git a/dockers/docker-base-buster/sources.list b/dockers/docker-base-buster/sources.list index 4a94e3161ab..8bb7d918fd5 100644 --- a/dockers/docker-base-buster/sources.list +++ b/dockers/docker-base-buster/sources.list @@ -6,3 +6,5 @@ deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster mai deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.arm64 b/dockers/docker-base-buster/sources.list.arm64 index d4450c25952..ac5da46fec9 100644 --- a/dockers/docker-base-buster/sources.list.arm64 +++ b/dockers/docker-base-buster/sources.list.arm64 @@ -6,3 +6,5 @@ deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free deb [arch=arm64] http://deb.debian.org/debian/ buster-backports main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.armhf b/dockers/docker-base-buster/sources.list.armhf index a053a509e14..17f76580523 100644 --- a/dockers/docker-base-buster/sources.list.armhf +++ b/dockers/docker-base-buster/sources.list.armhf @@ -6,3 +6,5 @@ deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free deb [arch=armhf] http://deb.debian.org/debian/ buster-backports main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free diff --git a/files/build/scripts/apt-get b/files/build/scripts/apt-get index 99b266f6f0f..aa0df769a3e 100755 --- a/files/build/scripts/apt-get +++ b/files/build/scripts/apt-get @@ -3,7 +3,7 @@ INSTALL= . /usr/local/share/buildinfo/scripts/buildinfo_base.sh -VERSION_FILE="/usr/local/share/buildinfo/base-versions/versions-deb" +VERSION_FILE="/usr/local/share/buildinfo/versions/versions-deb" if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then for para in $@ do @@ -13,17 +13,11 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then if [ ! -z "$INSTALL" ]; then if [[ "$para" == *=* ]]; then continue - package=$(echo "$para" | cut -d= -f1) - version=$(echo "$para" | cut -d= -f2) - if grep "^${package}=" $VERSION_FILE; then - if ! grep "^${package}==${version}$" $VERSION_FILE; then - EXPECTED_VERSION=$(grep "^${package}=" $VERSION_FILE | cut -d= -f3) - echo "${package}=${version}, expected version is ${EXPECTED_VERSION}" - exit 1 - fi - fi + elif [[ "$para" == *=* ]]; then + continue else - if ! grep "^${package}=" $VERSION_FILE; then + package=$para + if ! grep -q "^${package}=" $VERSION_FILE; then echo "The version of the package ${package} is not specified." exit 1 fi diff --git a/files/build/scripts/buildinfo_base.sh b/files/build/scripts/buildinfo_base.sh index b9c3f688e69..0cfe78300c8 100755 --- a/files/build/scripts/buildinfo_base.sh +++ b/files/build/scripts/buildinfo_base.sh @@ -32,6 +32,6 @@ check_version_control() ENABLE_VERSION_CONTROL_DEB=$(check_version_control "deb") ENABLE_VERSION_CONTROL_PY2=$(check_version_control "py2") ENABLE_VERSION_CONTROL_PY3=$(check_version_control "py3") -ENABLE_VERSION_CONTROL_WEB=$(check_version_control "web") +ENABLE_VERSION_CONTROL_WEB=$(check_version_control "url") ENABLE_VERSION_CONTROL_GIT=$(check_version_control "git") -ENABLE_VERSION_CONTROL_DCK=$(check_version_control "dck") +ENABLE_VERSION_CONTROL_DCK=$(check_version_control "docker") diff --git a/files/build/scripts/wget b/files/build/scripts/wget index 40675992c84..156b5101aa9 100755 --- a/files/build/scripts/wget +++ b/files/build/scripts/wget @@ -4,7 +4,6 @@ COMPONENT=web VERSION_FILE=$VERSION_PATH/versions-web BUILD_VERSION_FILE=$BUILD_VERSION_PATH/versions-web -ENABLE_VERSION_CONTROL=$(check_version_control "$COMPONENT") [ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/wget PARAS=("$@") @@ -73,10 +72,10 @@ process() fi real_version=$(get_version "$package_url") - if [ "$ENABLE_VERSION_CONTROL" == "y" ] && [ "$version" != "$real_version" ]; then + if [ "$ENABLE_VERSION_CONTROL_WEB" == "y" ] && [ "$version" != "$real_version" ]; then return 1 fi - elif [ "$ENABLE_VERSION_CONTROL" == "y" ]; then + elif [ "$ENABLE_VERSION_CONTROL_WEB" == "y" ]; then return 1 fi @@ -85,7 +84,7 @@ process() echo "$package_url==$real_version" >> ${BUILD_VERSION_FILE} # If the version is mismatched, exit the wget/curl - if [ "$ENABLE_VERSION_CONTROL" == "y" ] && [ "$version" != "$real_version" ]; then + if [ "$ENABLE_VERSION_CONTROL_WEB" == "y" ] && [ "$version" != "$real_version" ]; then return 1 fi diff --git a/rules/config b/rules/config index 611f7ca9889..634719c6b3b 100644 --- a/rules/config +++ b/rules/config @@ -169,13 +169,13 @@ PACKAGE_URL_PREFIX ?= https://packages.trafficmanager.net/public/packages # TRUSTED_GPG_URLS - the trusted gpgs, separated by comma TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,https://packages.microsoft.com/keys/microsoft.asc -# SONIC_VERSION_CONTROL_COMPONENTS - Valid values: none|all|components..., the components consist of one or multiple: deb,py2,py3,web,git,dck, seperated by comma +# SONIC_VERSION_CONTROL_COMPONENTS - Valid values: none|all|components..., the components consist of one or multiple: deb,py2,py3,url,git,docker, seperated by comma # none : disable the version control # all : enable the version control for all components # deb : debian packages # py2 : python2 packages # py3 : python3 pakcages -# web : web packages, downloaded by wget, curl +# url : web packages, downloaded by wget, curl # git : git repositories, donloaded by git clone -# dck : docker base images +# docker: docker base images SONIC_VERSION_CONTROL_COMPONENTS ?= none diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 1a8087cfc40..45faf88b596 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -6,6 +6,7 @@ ARCH=$3 DOCKERFILE_TARGE=$4 DISTRO=$5 +[ -z "$BUILD_SLAVE" ] && BUILD_SLAVE=n [ -z "$DOCKERFILE_TARGE" ] && DOCKERFILE_TARGE=$DOCKERFILE DOCKERFILE_PATH=$(dirname "$DOCKERFILE_TARGE") BUILDINFO_PATH="${DOCKERFILE_PATH}/buildinfo" @@ -27,11 +28,6 @@ ENV OLDPATH=$PATH ENV PATH="/usr/local/share/buildinfo/scripts:$PATH" RUN pre_run_buildinfo' -DOCKERFILE_POST_SCRIPT="RUN post_run_buildinfo" -[ "$BUILD_SLAVE" != "y" ] && DOCKERFILE_POST_SCRIPT="$DOCKERFILE_POST_SCRIPT -ENV PATH=\$OLDPATH" - - # Add the auto-generate code if it is not added in the target Dockerfile if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKERFILE_TARGE; then # Insert the docker build script before the RUN command @@ -41,6 +37,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE # Append the docker build script at the end of the docker file echo "RUN post_run_buildinfo" >> $TEMP_FILE + [ -z "$BUILD_SLAVE" ] && BUILD_SLAVE=n [ "$BUILD_SLAVE" != "y" ] && echo "ENV PATH=\$OLDPATH" >> $TEMP_FILE cat $TEMP_FILE > $DOCKERFILE_TARGE diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py index 0d82955990e..6b4614d48a5 100755 --- a/scripts/versions_manager.py +++ b/scripts/versions_manager.py @@ -84,6 +84,8 @@ def dump_to_path(self, file_path, config=False, priority=999): os.makedirs(file_path) filename = self.get_filename() if config and self.ctype == 'deb': + none_config_file_path = os.path.join(file_path, filename) + self.dump_to_file(none_config_file_path, False, priority) filename = VERSION_DEB_PREFERENCE file_path = os.path.join(file_path, filename) self.dump_to_file(file_path, config, priority) @@ -135,9 +137,10 @@ def get_order_keys(self): arch = '' return (self.ctype, dist, arch) - def clean_info(self, clean_dist=True, clean_arch=True): - if clean_dist and self.ctype != 'deb': - self.dist = ALL_DIST + def clean_info(self, clean_dist=True, clean_arch=True, force=False): + if clean_dist: + if force or self.ctype != 'deb': + self.dist = ALL_DIST if clean_arch: self.arch = ALL_ARCH @@ -296,9 +299,9 @@ def filter(self, ctypes=[]): components.append(component) self.components = components - def clean_info(self, clean_dist=True, clean_arch=True): + def clean_info(self, clean_dist=True, clean_arch=True, force=False): for component in self.components: - component.clean_info(clean_dist=clean_dist, clean_arch=clean_arch) + component.clean_info(clean_dist=clean_dist, clean_arch=clean_arch, force=force) def clone(self, ctypes=None, exclude_ctypes=None): components = [] @@ -633,7 +636,7 @@ def generate(self): default_module = VersionModule() default_module.load(default_module_path, filter_dist=args.distribution, filter_arch=args.architecture) config = module.get_config_module(default_module, args.distribution, args.architecture) - config.clean_info() + config.clean_info(force=True) config.dump(args.target_path, config=True, priority=args.priority) if __name__ == "__main__": From 101593b06de73dee0b79e9252034bd8ff9f0516f Mon Sep 17 00:00:00 2001 From: xumia Date: Fri, 20 Nov 2020 08:22:35 +0000 Subject: [PATCH 04/18] Fix some issues 1. Change some code format issues 2. Fix curl calling wget command, pip2 calling pip3 issue 3. Fix wget/curl downloading multiple urls issue --- .gitignore | 7 +- Makefile.work | 6 +- build_debian.sh | 10 +- dockers/docker-base-buster/sources.list | 3 + dockers/docker-base-buster/sources.list.arm64 | 1 + dockers/docker-base-buster/sources.list.armhf | 1 + dockers/docker-base-stretch/sources.list | 3 + .../docker-base-stretch/sources.list.arm64 | 3 + .../docker-base-stretch/sources.list.armhf | 3 + files/build/scripts/apt-get | 2 +- files/build/scripts/buildinfo_base.sh | 119 +++++++++++++++++- files/build/scripts/curl | 7 +- files/build/scripts/pip | 58 +-------- files/build/scripts/pip2 | 12 +- files/build/scripts/pip3 | 12 +- files/build/scripts/wget | 91 +------------- rules/config | 4 +- ...e_image.sh => build_debian_base_system.sh} | 1 - slave.mk | 2 +- sonic-slave-stretch/Dockerfile.j2 | 1 - 20 files changed, 165 insertions(+), 181 deletions(-) mode change 100755 => 120000 files/build/scripts/pip rename scripts/{build_host_base_image.sh => build_debian_base_system.sh} (99%) diff --git a/.gitignore b/.gitignore index 111e73c2745..47674a3f05e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ target/ *-dbg *dbg.j2 *.img -**/buildinfo # Autogenerated Dockerfiles sonic-slave*/Dockerfile @@ -62,3 +61,9 @@ platform/broadcom/sonic-platform-modules-dell/s6100/modules/dell_s6100_lpc.c platform/broadcom/sonic-platform-modules-dell/z9100/modules/dell_ich.c platform/broadcom/sonic-platform-modules-dell/z9100/modules/dell_mailbox.c platform/broadcom/sonic-platform-modules-dell/z9264f/sonic_platform/ipmihelper.py + +# buildinfo +files/build/buildinfo +dockers/**/buildinfo +platform/**/buildinfo +sonic-slave*/**/buildinfo diff --git a/Makefile.work b/Makefile.work index e244d09728f..ba67d6329e5 100644 --- a/Makefile.work +++ b/Makefile.work @@ -249,12 +249,12 @@ endif $(DOCKER_BUILD) ; } ifeq "$(KEEP_SLAVE_ON)" "yes" ifdef SOURCE_FOLDER - @$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh $$?; /bin/bash" + @$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash" else - @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh $$?; /bin/bash" + @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash" endif else - @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh $$?" + @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?" endif sonic-slave-base-build : diff --git a/build_debian.sh b/build_debian.sh index 1f1b6c4ec8d..b168fb76e21 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -70,17 +70,13 @@ pushd $FILESYSTEM_ROOT sudo mount --bind . . popd -## Build the host base debian system -echo '[INFO] Build host base image...' -TARGET_PATH=$TARGET_PATH scripts/build_host_base_image.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT +## Build the host debian base system +echo '[INFO] Build host debian base system...' +TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT # Prepare buildinfo sudo scripts/prepare_base_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy -# Generate version files for apt/pip/pip3 packages -# sudo LANG=C chroot $FILESYSTEM_ROOT generate_version_files - - ## Config hostname and hosts, otherwise 'sudo ...' will complain 'sudo: unable to resolve host ...' sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo '$HOSTNAME' > /etc/hostname" sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo '127.0.0.1 $HOSTNAME' >> /etc/hosts" diff --git a/dockers/docker-base-buster/sources.list b/dockers/docker-base-buster/sources.list index 8bb7d918fd5..0eef72d9fa2 100644 --- a/dockers/docker-base-buster/sources.list +++ b/dockers/docker-base-buster/sources.list @@ -6,5 +6,8 @@ deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster mai deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free + +# Debian mirror supports multiple versions for a package deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.arm64 b/dockers/docker-base-buster/sources.list.arm64 index ac5da46fec9..6375734e99e 100644 --- a/dockers/docker-base-buster/sources.list.arm64 +++ b/dockers/docker-base-buster/sources.list.arm64 @@ -8,3 +8,4 @@ deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non- deb [arch=arm64] http://deb.debian.org/debian/ buster-backports main contrib non-free deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster main contrib non-free deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-buster/sources.list.armhf b/dockers/docker-base-buster/sources.list.armhf index 17f76580523..a03af1a33ac 100644 --- a/dockers/docker-base-buster/sources.list.armhf +++ b/dockers/docker-base-buster/sources.list.armhf @@ -8,3 +8,4 @@ deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non- deb [arch=armhf] http://deb.debian.org/debian/ buster-backports main contrib non-free deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster main contrib non-free deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian buster-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list b/dockers/docker-base-stretch/sources.list index 752b1bb4fc6..0c29b339bb8 100644 --- a/dockers/docker-base-stretch/sources.list +++ b/dockers/docker-base-stretch/sources.list @@ -6,3 +6,6 @@ deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch ma deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ stretch/updates main contrib non-free deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ stretch-backports main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free +deb [arch=amd64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list.arm64 b/dockers/docker-base-stretch/sources.list.arm64 index f5fe1d39142..520c4651991 100644 --- a/dockers/docker-base-stretch/sources.list.arm64 +++ b/dockers/docker-base-stretch/sources.list.arm64 @@ -6,3 +6,6 @@ deb-src [arch=arm64] http://deb.debian.org/debian stretch main contrib non-free deb [arch=arm64] http://security.debian.org stretch/updates main contrib non-free deb-src [arch=arm64] http://security.debian.org stretch/updates main contrib non-free deb [arch=arm64] http://deb.debian.org/debian/ stretch-backports main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free +deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/dockers/docker-base-stretch/sources.list.armhf b/dockers/docker-base-stretch/sources.list.armhf index 90c4d351f09..58077f31042 100644 --- a/dockers/docker-base-stretch/sources.list.armhf +++ b/dockers/docker-base-stretch/sources.list.armhf @@ -6,3 +6,6 @@ deb-src [arch=armhf] http://deb.debian.org/debian stretch main contrib non-free deb [arch=armhf] http://security.debian.org stretch/updates main contrib non-free deb-src [arch=armhf] http://security.debian.org stretch/updates main contrib non-free deb [arch=armhf] http://deb.debian.org/debian/ stretch-backports main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free +deb [arch=armhf] http://packages.trafficmanager.net/debian/debian stretch-backports main contrib non-free diff --git a/files/build/scripts/apt-get b/files/build/scripts/apt-get index aa0df769a3e..55283630393 100755 --- a/files/build/scripts/apt-get +++ b/files/build/scripts/apt-get @@ -29,4 +29,4 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then fi -/usr/bin/apt-get $@ +/usr/bin/apt-get "$@" diff --git a/files/build/scripts/buildinfo_base.sh b/files/build/scripts/buildinfo_base.sh index 0cfe78300c8..b20e953f050 100755 --- a/files/build/scripts/buildinfo_base.sh +++ b/files/build/scripts/buildinfo_base.sh @@ -9,6 +9,8 @@ DIFF_VERSION_PATH=$BUILDINFO_PATH/diff-versions BUILD_VERSION_PATH=$BUILDINFO_PATH/build-versions POST_VERSION_PATH=$BUILDINFO_PATH/post-versions VERSION_DEB_PREFERENCE=$BUILDINFO_PATH/versions/01-versions-deb +WEB_VERSION_FILE=$VERSION_PATH/versions-web +BUILD_WEB_VERSION_FILE=$BUILD_VERSION_PATH/versions-web . $BUILDINFO_PATH/config/buildinfo.config @@ -29,9 +31,122 @@ check_version_control() fi } +get_url_version() +{ + local package_url=$1 + /usr/bin/curl -ks $package_url | md5sum | cut -d' ' -f1 +} + +check_if_url_exist() +{ + local url=$1 + if /usr/bin/curl --output /dev/null --silent --head --fail "$1" > /dev/null 2>&1; then + echo y + else + echo n + fi +} + +verify_and_set_proxy_url() +{ + local index=$1 + local url=$2 + local version= + [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}') + if [ "$ENABLE_VERSION_CONTROL_WEB" != y ]; then + local real_version=$(get_url_version $url) + mkdir -p $LOG_PATH > /dev/null 2>&1 + echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE} + else + # When version control is enabled, check the version file, and set the proxy url + # If the proxy is not ready, it will verify the version of the current url + if [ -z "$version" ]; then + echo "Failed to verify the package: $url, the version is not specified" 2>&1 + exit 1 + fi + + local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) + local version_filename="${filename}-${version}" + local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}" + local url_exist=$(check_if_url_exist $proxy_url) + if [ "$url_exist" == y ]; then + COMMAND_PARAMETERS[$index]=$proxy_url + else + local real_version=$(get_url_version $url) + if [ "$real_version" != "$version" ]; then + echo "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2 + exit 1 + fi + fi + fi +} + +download_packages() +{ + COMMAND_PARAMETERS=("$@") + for (( i=0; i<${#COMMAND_PARAMETERS[@]}; i++ )) + do + local para=${COMMAND_PARAMETERS[$i]} + if [[ "$para" == *://* ]]; then + verify_and_set_proxy_url $i "$para" + fi + done + + $REAL_COMMAND "${COMMAND_PARAMETERS[@]}" +} + +run_pip_command() +{ + COMMAND_PARAMETERS=("$@") + + if [ ! -x "$REAL_COMMAND" ] && [ " $1" == "freeze" ]; then + return 1 + fi + + if [ "$ENABLE_VERSION_CONTROL_PY" != "y" ]; then + $REAL_COMMAND "$@" + return $? + fi + + local found=false + local install=false + local pip_version_file=$PIP_VERSION_FILE + local tmp_version_file=$(mktemp) + [ -f "$pip_version_file" ] && cp -f $pip_version_file $tmp_version_file + for para in "${COMMAND_PARAMETERS[@]}" + do + ([ "$para" == "-c" ] || [ "$para" == "--constraint" ]) && found=true + if [ "$para" == "install" ]; then + install=true + elif [[ "$para" == *.whl ]]; then + package_name=$(echo $para | cut -d- -f1 | tr _ .) + sed "/^${package_name}==/d" -i $tmp_version_file + elif [[ "$para" != -* ]]; then + package_name=$(echo $para | awk -F "[=><]" '{print $1}') + if ! grep -q "${package_name}==" $tmp_version_file; then + package_version=$(echo $para | awk -F "[=><]" '{for (i=1; i&2 + return 1 + fi + fi + fi + done + + if [ "$found" == "false" ] && [ "$install" == "true" ]; then + COMMAND_PARAMETERS+=("-c") + COMMAND_PARAMETERS+=("${tmp_version_file}") + fi + + $REAL_COMMAND "${COMMAND_PARAMETERS[@]}" + local result=$? + rm $tmp_version_file + return $result +} + ENABLE_VERSION_CONTROL_DEB=$(check_version_control "deb") ENABLE_VERSION_CONTROL_PY2=$(check_version_control "py2") ENABLE_VERSION_CONTROL_PY3=$(check_version_control "py3") -ENABLE_VERSION_CONTROL_WEB=$(check_version_control "url") +ENABLE_VERSION_CONTROL_WEB=$(check_version_control "web") ENABLE_VERSION_CONTROL_GIT=$(check_version_control "git") -ENABLE_VERSION_CONTROL_DCK=$(check_version_control "docker") +ENABLE_VERSION_CONTROL_DOCKER=$(check_version_control "docker") diff --git a/files/build/scripts/curl b/files/build/scripts/curl index b13573535b8..58bfe0395e5 100755 --- a/files/build/scripts/curl +++ b/files/build/scripts/curl @@ -1,7 +1,6 @@ #!/bin/bash +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh +[ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/curl -PARENT_PATH=$(dirname $0) -REAL_COMMAND=/usr/bin/curl - -REAL_COMMAND=$REAL_COMMAND ${PARENT_PATH}/wget $@ +REAL_COMMAND=$REAL_COMMAND download_packages "$@" diff --git a/files/build/scripts/pip b/files/build/scripts/pip deleted file mode 100755 index cb9c4e335d0..00000000000 --- a/files/build/scripts/pip +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -. /usr/local/share/buildinfo/scripts/buildinfo_base.sh - - -if [ -z "$PIP_COMMAND" ]; then - [ -x /usr/bin/pip ] && PIP_COMMAND=/usr/bin/pip - [ -x /usr/local/bin/pip ] && PIP_COMMAND=/usr/local/bin/pip -fi - -if [ ! -x "$PIP_COMMAND" ]; then - echo "The command pip not found" 1>&2 - exit 1 -fi - -if [ -z "$PIP_COMPONENT" ]; then - PIP_COMPONENT=py2 - if $PIP_COMMAND --version | grep -q "python 3"; then - PIP_COMPONENT=py3 - fi -fi - -ENABLE_VERSION_CONTROL=ENABLE_VERSION_CONTROL_PY2 -[ "$PIP_COMPONENT" == "py3" ] && ENABLE_VERSION_CONTROL=ENABLE_VERSION_CONTROL_PY3 -VERSION_FILE="$BUILDINFO_PATH/versions/versions-${PIP_COMPONENT}" - -if [ "$ENABLE_VERSION_CONTROL" != "y" ]; then - $PIP_COMMAND "$@" - exit $? -fi - -paras=("$@") -FOUND=false -INSTALL=false -VERSION_CONFIG_FILE="${VERSION_FILE}.config" -cp -f $VERSION_FILE $VERSION_CONFIG_FILE -for para in "${paras[@]}" -do - ([ "$para" == "-c" ] || [ "$para" == "--constraint" ]) && FOUND=true - [ "$para" == "install" ] && INSTALL=true - if [[ "$para" == *.whl ]]; then - package_name=$(echo $para | cut -d- -f1 | tr _ .) - sed "/^${package_name}==/d" -i $VERSION_CONFIG_FILE - fi -done - -if [ "$FOUND" == "false" ] && [ "$INSTALL" == "true" ]; then - paras+=("-c") - paras+=("${VERSION_CONFIG_FILE}") -fi - - -if [ ! -x "$PIP_COMMAND" ] && [ " $1" == "freeze" ]; then - exit 1 -fi - -$PIP_COMMAND ${paras[@]} diff --git a/files/build/scripts/pip b/files/build/scripts/pip new file mode 120000 index 00000000000..b31c5d8ca9d --- /dev/null +++ b/files/build/scripts/pip @@ -0,0 +1 @@ +pip2 \ No newline at end of file diff --git a/files/build/scripts/pip2 b/files/build/scripts/pip2 index 42c65865fdf..0c09fc99ac2 100755 --- a/files/build/scripts/pip2 +++ b/files/build/scripts/pip2 @@ -1,13 +1,15 @@ #!/bin/bash -PARENT_PATH=$(dirname $0) +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh -PIP_COMMAND=/usr/bin/pip2 -[ -x /usr/local/bin/pip2 ] && PIP_COMMAND=/usr/local/bin/pip2 +VERSION_FILE="$BUILDINFO_PATH/versions/versions-py2" +REAL_COMMAND=/usr/bin/pip2 +[ -x /usr/local/bin/pip2 ] && REAL_COMMAND=/usr/local/bin/pip2 -if [ ! -x "$PIP_COMMAND" ]; then +if [ ! -x "$REAL_COMMAND" ]; then echo "The command pip2 not found" 1>&2 exit 1 fi -PIP_COMPONENT=py2 PIP_COMMAND="$PIP_COMMAND" ${PARENT_PATH}/pip $@ +PIP_VERSION_FILE=$VERSION_FILE ENABLE_VERSION_CONTROL_PY=$ENABLE_VERSION_CONTROL_PY2 REAL_COMMAND=$REAL_COMMAND run_pip_command "$@" + diff --git a/files/build/scripts/pip3 b/files/build/scripts/pip3 index 15788a2e1e4..9785b2d9650 100755 --- a/files/build/scripts/pip3 +++ b/files/build/scripts/pip3 @@ -1,13 +1,13 @@ #!/bin/bash -PARENT_PATH=$(dirname $0) +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh +REAL_COMMAND=/usr/bin/pip3 +[ -x /usr/local/bin/pip3 ] && REAL_COMMAND=/usr/local/bin/pip3 -PIP_COMMAND=/usr/bin/pip3 -[ -x /usr/local/bin/pip3 ] && PIP_COMMAND=/usr/local/bin/pip3 - -if [ ! -x "$PIP_COMMAND" ]; then +if [ ! -x "$REAL_COMMAND" ]; then echo "The command pip3 not found" 1>&2 exit 1 fi -PIP_COMPONENT=py2 PIP_COMMAND="$PIP_COMMAND" ${PARENT_PATH}/pip $@ +ENABLE_VERSION_CONTROL_PY=$ENABLE_VERSION_CONTROL_PY2 REAL_COMMAND=$REAL_COMMAND run_pip_command "$@" + diff --git a/files/build/scripts/wget b/files/build/scripts/wget index 156b5101aa9..6111f7ae40a 100755 --- a/files/build/scripts/wget +++ b/files/build/scripts/wget @@ -1,95 +1,6 @@ #!/bin/bash . /usr/local/share/buildinfo/scripts/buildinfo_base.sh -COMPONENT=web -VERSION_FILE=$VERSION_PATH/versions-web -BUILD_VERSION_FILE=$BUILD_VERSION_PATH/versions-web - [ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/wget -PARAS=("$@") - -if [ "$SKIP_BUILD_HOOK" == "y" ]; then - $REAL_COMMAND "$@" - exit $? -fi - -get_version() -{ - local package_url=$1 - if [[ "$REAL_COMMAND" == */curl ]]; then - $REAL_COMMAND "$package_url" | md5sum | cut -d' ' -f1 - else - $REAL_COMMAND -qO - "$package_url" | md5sum | cut -d' ' -f1 - fi -} - -get_weburl_index() -{ - for (( i=0; i<${#PARAS[@]}; i++ )) - do - local para=${PARAS[$i]} - if [[ "$para" == *://* ]]; then - echo $i - return - fi - done -} - -check_if_file_exist() -{ - if [[ "$REAL_COMMAND" == */curl ]]; then - $REAL_COMMAND --output /dev/null --silent --head --fail "$1" > /dev/null 2>&1 - else - $REAL_COMMAND -S --spider $1 2>&1 | grep -q "200 OK" - fi -} - -process() -{ - local index=$(get_weburl_index) - - if [ -z "$index" ]; then - log_err "Failed to parse the url" - return 1 - fi - local package_url=${PARAS[$index]} - local version="" - local real_version="" - [ -f $VERSION_FILE ] && version=$(grep "^${package_url}=" $VERSION_FILE | awk -F"==" '{print $NF}') - - if [ ! -z "$version" ]; then - local filename=$(echo $package_url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) - local version_filename="${filename}-${version}" - local url="${PACKAGE_URL_PREFIX}${version_filename}" - PARAS[$index]=$url - - # Try to download package from the proxy - if check_if_file_exist $url; then - ${REAL_COMMAND} ${PARAS[@]} - local result=$? - [ -f "$version_filename" ] && mv -f "${version_filename}" "${filename}" - return $result - fi - - real_version=$(get_version "$package_url") - if [ "$ENABLE_VERSION_CONTROL_WEB" == "y" ] && [ "$version" != "$real_version" ]; then - return 1 - fi - elif [ "$ENABLE_VERSION_CONTROL_WEB" == "y" ]; then - return 1 - fi - - [ -z "$real_version" ] && real_version=$(get_version "$package_url") - mkdir -p $LOG_PATH > /dev/null 2>&1 - echo "$package_url==$real_version" >> ${BUILD_VERSION_FILE} - - # If the version is mismatched, exit the wget/curl - if [ "$ENABLE_VERSION_CONTROL_WEB" == "y" ] && [ "$version" != "$real_version" ]; then - return 1 - fi - - ${REAL_COMMAND} "$@" -} - -process "$@" +REAL_COMMAND=$REAL_COMMAND download_packages "$@" diff --git a/rules/config b/rules/config index 634719c6b3b..fdad3dc44bc 100644 --- a/rules/config +++ b/rules/config @@ -169,13 +169,13 @@ PACKAGE_URL_PREFIX ?= https://packages.trafficmanager.net/public/packages # TRUSTED_GPG_URLS - the trusted gpgs, separated by comma TRUSTED_GPG_URLS = https://packages.trafficmanager.net/debian/public_key.gpg,https://packages.microsoft.com/keys/microsoft.asc -# SONIC_VERSION_CONTROL_COMPONENTS - Valid values: none|all|components..., the components consist of one or multiple: deb,py2,py3,url,git,docker, seperated by comma +# SONIC_VERSION_CONTROL_COMPONENTS - Valid values: none|all|components..., the components consist of one or multiple: deb,py2,py3,web,git,docker, seperated by comma # none : disable the version control # all : enable the version control for all components # deb : debian packages # py2 : python2 packages # py3 : python3 pakcages -# url : web packages, downloaded by wget, curl +# web : web packages, downloaded by wget, curl # git : git repositories, donloaded by git clone # docker: docker base images SONIC_VERSION_CONTROL_COMPONENTS ?= none diff --git a/scripts/build_host_base_image.sh b/scripts/build_debian_base_system.sh similarity index 99% rename from scripts/build_host_base_image.sh rename to scripts/build_debian_base_system.sh index e3ec1dcf669..e1539df05db 100755 --- a/scripts/build_host_base_image.sh +++ b/scripts/build_debian_base_system.sh @@ -1,6 +1,5 @@ #!/bin/bash - CONFIGURED_ARCH=$1 IMAGE_DISTRO=$2 FILESYSTEM_ROOT=$3 diff --git a/slave.mk b/slave.mk index bf1bd04508a..621d098af61 100644 --- a/slave.mk +++ b/slave.mk @@ -102,7 +102,7 @@ include $(RULES_PATH)/config ############################################################################### -## Version control relatived exports +## Version control related exports ############################################################################### export PACKAGE_URL_PREFIX export TRUSTED_GPG_URLS diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index 3827face126..b6927befc11 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -41,7 +41,6 @@ RUN echo "deb [arch=arm64] http://deb.debian.org/debian stretch main contrib non echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian stretch-updates main contrib non-free" >> /etc/apt/sources.list {%- endif %} - ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive From a7df576bfaae37cb9ae38c75c2a53d2a9bd59a57 Mon Sep 17 00:00:00 2001 From: xumia Date: Fri, 20 Nov 2020 09:30:49 +0000 Subject: [PATCH 05/18] Fix some code format issue --- Makefile | 1 + Makefile.work | 8 +++++++- scripts/versions_manager.py | 3 --- sonic-slave-buster/Dockerfile.j2 | 1 - 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 8bb8f443aef..12e760f6dd6 100644 --- a/Makefile +++ b/Makefile @@ -35,5 +35,6 @@ ifeq ($(NOSTRETCH), 0) endif BLDENV=buster make -f Makefile.work $@ +# Freeze the versions, see more detail options: scripts/versions_manager.py freeze -h freeze: @scripts/versions_manager.py freeze $(OPTIONS) diff --git a/Makefile.work b/Makefile.work index ba67d6329e5..7f4a4d7ada1 100644 --- a/Makefile.work +++ b/Makefile.work @@ -93,11 +93,17 @@ include rules/config SLAVE_BASE_IMAGE = $(SLAVE_DIR) SLAVE_IMAGE = $(SLAVE_BASE_IMAGE)-$(USER) + +# Generate the version control build info $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \ TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ scripts/generate_buildinfo_config.sh) + +# Generate the slave Dockerfile, and prepare build info for it $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile) $(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV)) + +# Add the versions in the tag, if the version change, need to rebuild the slave SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}') SLAVE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile.user $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* | sha1sum | awk '{print substr($$1,0,11);}') @@ -249,7 +255,7 @@ endif $(DOCKER_BUILD) ; } ifeq "$(KEEP_SLAVE_ON)" "yes" ifdef SOURCE_FOLDER - @$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash" + @$(DOCKER_RUN) -v $(SOURCE_FOLDER):/var/$(USER)/src $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash" else @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?; /bin/bash" endif diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py index 6b4614d48a5..4dc4c51308b 100755 --- a/scripts/versions_manager.py +++ b/scripts/versions_manager.py @@ -611,9 +611,6 @@ def merge(self): module.dump(args.target_path) def generate(self): - script_path = os.path.dirname(sys.argv[0]) - root_path = os.path.dirname(script_path) - parser = argparse.ArgumentParser(description = 'Generate the version files') parser.add_argument('-t', '--target_path', required=True, help='target path to generate the version lock files') group = parser.add_mutually_exclusive_group(required=True) diff --git a/sonic-slave-buster/Dockerfile.j2 b/sonic-slave-buster/Dockerfile.j2 index e9bb9b21dd1..538c6f3f257 100644 --- a/sonic-slave-buster/Dockerfile.j2 +++ b/sonic-slave-buster/Dockerfile.j2 @@ -42,7 +42,6 @@ RUN echo "deb [arch=arm64] http://deb.debian.org/debian buster main contrib non- echo "deb [arch=arm64] http://packages.trafficmanager.net/debian/debian buster-updates main contrib non-free" >> /etc/apt/sources.list {%- endif %} - ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive From fa2bc6ef335672661189d6d7ca7b20f354d21678 Mon Sep 17 00:00:00 2001 From: xumia Date: Sun, 22 Nov 2020 04:03:27 +0000 Subject: [PATCH 06/18] Fix bug --- files/build/scripts/buildinfo_base.sh | 111 +++++++++++++------------- files/build/scripts/pip2 | 1 - files/build/scripts/pip3 | 5 +- files/build/scripts/wget | 5 ++ 4 files changed, 62 insertions(+), 60 deletions(-) diff --git a/files/build/scripts/buildinfo_base.sh b/files/build/scripts/buildinfo_base.sh index b20e953f050..289ee634328 100755 --- a/files/build/scripts/buildinfo_base.sh +++ b/files/build/scripts/buildinfo_base.sh @@ -47,57 +47,63 @@ check_if_url_exist() fi } -verify_and_set_proxy_url() +download_packages() { - local index=$1 - local url=$2 - local version= - [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}') - if [ "$ENABLE_VERSION_CONTROL_WEB" != y ]; then - local real_version=$(get_url_version $url) - mkdir -p $LOG_PATH > /dev/null 2>&1 - echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE} - else - # When version control is enabled, check the version file, and set the proxy url - # If the proxy is not ready, it will verify the version of the current url - if [ -z "$version" ]; then - echo "Failed to verify the package: $url, the version is not specified" 2>&1 - exit 1 - fi + local parameters=("$@") + local filenames= + declare -A filenames + for (( i=0; i<${#parameters[@]}; i++ )) + do + local para=${parameters[$i]} + local nexti=$((i+1)) + if [[ "$para" == *://* ]]; then + local url=$para + local real_version= + if [ "$ENABLE_VERSION_CONTROL_WEB" == y ]; then + local version= + local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) + [ -f $WEB_VERSION_FILE ] && version=$(grep "^${url}=" $WEB_VERSION_FILE | awk -F"==" '{print $NF}') + if [ -z "$version" ]; then + echo "Failed to verify the package: $url, the version is not specified" 2>&1 + exit 1 + fi - local filename=$(echo $url | awk -F"/" '{print $NF}' | cut -d? -f1 | cut -d# -f1) - local version_filename="${filename}-${version}" - local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}" - local url_exist=$(check_if_url_exist $proxy_url) - if [ "$url_exist" == y ]; then - COMMAND_PARAMETERS[$index]=$proxy_url - else - local real_version=$(get_url_version $url) - if [ "$real_version" != "$version" ]; then - echo "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2 - exit 1 + local version_filename="${filename}-${version}" + local proxy_url="${PACKAGE_URL_PREFIX}/${version_filename}" + local url_exist=$(check_if_url_exist $proxy_url) + if [ "$url_exist" == y ]; then + parameters[$i]=$proxy_url + filenames[$version_filename]=$filename + real_version=$version + else + real_version=$(get_url_version $url) + if [ "$real_version" != "$version" ]; then + echo "Failed to verify url: $url, real hash value: $real_version, expected value: $version_filename" 1>&2 + exit 1 + fi + fi + else + real_version=$(get_url_version $url) fi + + echo "$url==$real_version" >> ${BUILD_WEB_VERSION_FILE} fi - fi -} + done -download_packages() -{ - COMMAND_PARAMETERS=("$@") - for (( i=0; i<${#COMMAND_PARAMETERS[@]}; i++ )) + $REAL_COMMAND "${parameters[@]}" + local result=$? + + for filename in "${!filenames[@]}" do - local para=${COMMAND_PARAMETERS[$i]} - if [[ "$para" == *://* ]]; then - verify_and_set_proxy_url $i "$para" - fi + [ -f "$filename" ] && mv "$filename" "${filenames[$filename]}" done - $REAL_COMMAND "${COMMAND_PARAMETERS[@]}" + return $result } run_pip_command() { - COMMAND_PARAMETERS=("$@") + parameters=("$@") if [ ! -x "$REAL_COMMAND" ] && [ " $1" == "freeze" ]; then return 1 @@ -108,37 +114,28 @@ run_pip_command() return $? fi - local found=false - local install=false + local found=n + local install=n local pip_version_file=$PIP_VERSION_FILE local tmp_version_file=$(mktemp) [ -f "$pip_version_file" ] && cp -f $pip_version_file $tmp_version_file - for para in "${COMMAND_PARAMETERS[@]}" + for para in "${parameters[@]}" do - ([ "$para" == "-c" ] || [ "$para" == "--constraint" ]) && found=true + ([ "$para" == "-c" ] || [ "$para" == "--constraint" ]) && found=y if [ "$para" == "install" ]; then - install=true + install=y elif [[ "$para" == *.whl ]]; then package_name=$(echo $para | cut -d- -f1 | tr _ .) sed "/^${package_name}==/d" -i $tmp_version_file - elif [[ "$para" != -* ]]; then - package_name=$(echo $para | awk -F "[=><]" '{print $1}') - if ! grep -q "${package_name}==" $tmp_version_file; then - package_version=$(echo $para | awk -F "[=><]" '{for (i=1; i&2 - return 1 - fi - fi fi done - if [ "$found" == "false" ] && [ "$install" == "true" ]; then - COMMAND_PARAMETERS+=("-c") - COMMAND_PARAMETERS+=("${tmp_version_file}") + if [ "$found" == "n" ] && [ "$install" == "y" ]; then + parameters+=("-c") + parameters+=("${tmp_version_file}") fi - $REAL_COMMAND "${COMMAND_PARAMETERS[@]}" + $REAL_COMMAND "${parameters[@]}" local result=$? rm $tmp_version_file return $result diff --git a/files/build/scripts/pip2 b/files/build/scripts/pip2 index 0c09fc99ac2..6116d8121a5 100755 --- a/files/build/scripts/pip2 +++ b/files/build/scripts/pip2 @@ -12,4 +12,3 @@ if [ ! -x "$REAL_COMMAND" ]; then fi PIP_VERSION_FILE=$VERSION_FILE ENABLE_VERSION_CONTROL_PY=$ENABLE_VERSION_CONTROL_PY2 REAL_COMMAND=$REAL_COMMAND run_pip_command "$@" - diff --git a/files/build/scripts/pip3 b/files/build/scripts/pip3 index 9785b2d9650..c667f7576ea 100755 --- a/files/build/scripts/pip3 +++ b/files/build/scripts/pip3 @@ -1,6 +1,8 @@ #!/bin/bash . /usr/local/share/buildinfo/scripts/buildinfo_base.sh + +VERSION_FILE="$BUILDINFO_PATH/versions/versions-py3" REAL_COMMAND=/usr/bin/pip3 [ -x /usr/local/bin/pip3 ] && REAL_COMMAND=/usr/local/bin/pip3 @@ -9,5 +11,4 @@ if [ ! -x "$REAL_COMMAND" ]; then exit 1 fi -ENABLE_VERSION_CONTROL_PY=$ENABLE_VERSION_CONTROL_PY2 REAL_COMMAND=$REAL_COMMAND run_pip_command "$@" - +PIP_VERSION_FILE=$VERSION_FILE ENABLE_VERSION_CONTROL_PY=$ENABLE_VERSION_CONTROL_PY2 REAL_COMMAND=$REAL_COMMAND run_pip_command "$@" diff --git a/files/build/scripts/wget b/files/build/scripts/wget index 6111f7ae40a..bd575ca616a 100755 --- a/files/build/scripts/wget +++ b/files/build/scripts/wget @@ -3,4 +3,9 @@ . /usr/local/share/buildinfo/scripts/buildinfo_base.sh [ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/wget +if [ "$SKIP_BUILD_HOOK" == y ]; then + $REAL_COMMAND "$@" + exit $? +fi + REAL_COMMAND=$REAL_COMMAND download_packages "$@" From 49a1ec9ee4f8500c404080e2daa11604a9c23b7c Mon Sep 17 00:00:00 2001 From: xumia Date: Tue, 24 Nov 2020 13:40:06 +0000 Subject: [PATCH 07/18] Fix bug --- files/build/versions/default/readme.md | 1 - scripts/prepare_docker_buildinfo.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 files/build/versions/default/readme.md diff --git a/files/build/versions/default/readme.md b/files/build/versions/default/readme.md deleted file mode 100644 index 438972e1906..00000000000 --- a/files/build/versions/default/readme.md +++ /dev/null @@ -1 +0,0 @@ -The default versions diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 45faf88b596..5050d5cd8c9 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -48,7 +48,7 @@ fi cp -rf files/build/buildinfo/* $BUILDINFO_PATH # Copy the docker build info scirpts -cp -rf files/build/scripts "${BUILDINFO_PATH}/" +cp -r files/build/scripts "${BUILDINFO_PATH}/" # Build the slave running config if [ "$BUILD_SLAVE" == "y" ]; then From 42e39144a684b3dd0d5f5a689f7cf7600ed2b6ec Mon Sep 17 00:00:00 2001 From: xumia Date: Mon, 30 Nov 2020 02:01:12 +0000 Subject: [PATCH 08/18] Fix command path hard code in build info scripts issue --- files/build/scripts/apt-get | 8 +++++++- files/build/scripts/buildinfo_base.sh | 7 +++++++ files/build/scripts/pip2 | 3 +-- files/build/scripts/pip3 | 4 +--- files/build/scripts/wget | 6 +++++- 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/files/build/scripts/apt-get b/files/build/scripts/apt-get index 55283630393..9ae50a11865 100755 --- a/files/build/scripts/apt-get +++ b/files/build/scripts/apt-get @@ -3,6 +3,12 @@ INSTALL= . /usr/local/share/buildinfo/scripts/buildinfo_base.sh +REAL_COMMAND=$(get_command apt-get) +if [ -z "$REAL_COMMAND" ]; then + echo "The command apt-get does not exist." 1>&2 + exit 1 +fi + VERSION_FILE="/usr/local/share/buildinfo/versions/versions-deb" if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then for para in $@ @@ -29,4 +35,4 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then fi -/usr/bin/apt-get "$@" +$REAL_COMMAND "$@" diff --git a/files/build/scripts/buildinfo_base.sh b/files/build/scripts/buildinfo_base.sh index 289ee634328..7f9de6e436c 100755 --- a/files/build/scripts/buildinfo_base.sh +++ b/files/build/scripts/buildinfo_base.sh @@ -22,6 +22,13 @@ log_err() echo "$1" 1>&2 } +get_command() +{ + local path=$(echo $PATH | sed 's#[^:]*buildinfo/scripts:##') + local command=$(PATH=$path which $1) + echo $command +} + check_version_control() { if [[ ",$SONIC_VERSION_CONTROL_COMPONENTS," == *,all,* ]] || [[ ",$SONIC_VERSION_CONTROL_COMPONENTS," == *,$1,* ]]; then diff --git a/files/build/scripts/pip2 b/files/build/scripts/pip2 index 6116d8121a5..e2ccfff2aac 100755 --- a/files/build/scripts/pip2 +++ b/files/build/scripts/pip2 @@ -3,8 +3,7 @@ . /usr/local/share/buildinfo/scripts/buildinfo_base.sh VERSION_FILE="$BUILDINFO_PATH/versions/versions-py2" -REAL_COMMAND=/usr/bin/pip2 -[ -x /usr/local/bin/pip2 ] && REAL_COMMAND=/usr/local/bin/pip2 +REAL_COMMAND=$(get_command pip2) if [ ! -x "$REAL_COMMAND" ]; then echo "The command pip2 not found" 1>&2 diff --git a/files/build/scripts/pip3 b/files/build/scripts/pip3 index c667f7576ea..36a8fd93a13 100755 --- a/files/build/scripts/pip3 +++ b/files/build/scripts/pip3 @@ -3,9 +3,7 @@ . /usr/local/share/buildinfo/scripts/buildinfo_base.sh VERSION_FILE="$BUILDINFO_PATH/versions/versions-py3" -REAL_COMMAND=/usr/bin/pip3 -[ -x /usr/local/bin/pip3 ] && REAL_COMMAND=/usr/local/bin/pip3 - +REAL_COMMAND=$(get_command pip3) if [ ! -x "$REAL_COMMAND" ]; then echo "The command pip3 not found" 1>&2 exit 1 diff --git a/files/build/scripts/wget b/files/build/scripts/wget index bd575ca616a..c4cb1a3d1be 100755 --- a/files/build/scripts/wget +++ b/files/build/scripts/wget @@ -1,7 +1,11 @@ #!/bin/bash . /usr/local/share/buildinfo/scripts/buildinfo_base.sh -[ -z $REAL_COMMAND ] && REAL_COMMAND=/usr/bin/wget +[ -z $REAL_COMMAND ] && REAL_COMMAND=$(get_command wget) +if [ -z "$REAL_COMMAND" ]; then + echo "The command wget does not exist." 1>&2 + exit 1 +fi if [ "$SKIP_BUILD_HOOK" == y ]; then $REAL_COMMAND "$@" From 5eb2d9f4e86ed15b29ac2ac29a549e5e5ef4c2a9 Mon Sep 17 00:00:00 2001 From: xumia Date: Sat, 5 Dec 2020 09:54:11 +0000 Subject: [PATCH 09/18] Add debian package sonic-build-tools --- .gitignore | 1 + Makefile | 2 +- Makefile.work | 17 ++++++++---- files/build/Makefile | 37 +++++++++++++++++++++++++ files/build/debian/control | 10 +++++++ files/build/{scripts => hooks}/apt-get | 0 files/build/{scripts => hooks}/curl | 0 files/build/{scripts => hooks}/pip | 0 files/build/{scripts => hooks}/pip2 | 0 files/build/{scripts => hooks}/pip3 | 0 files/build/{scripts => hooks}/wget | 0 files/build/scripts/buildinfo_base.sh | 2 +- files/build/scripts/pre_run_buildinfo | 1 + files/build/scripts/set_build_hooks | 32 +++++++++++++++++++++ scripts/collect_docker_version_files.sh | 7 ++++- scripts/prepare_base_image_buildinfo.sh | 1 + scripts/prepare_docker_buildinfo.sh | 9 ++---- 17 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 files/build/Makefile create mode 100644 files/build/debian/control rename files/build/{scripts => hooks}/apt-get (100%) rename files/build/{scripts => hooks}/curl (100%) rename files/build/{scripts => hooks}/pip (100%) rename files/build/{scripts => hooks}/pip2 (100%) rename files/build/{scripts => hooks}/pip3 (100%) rename files/build/{scripts => hooks}/wget (100%) create mode 100755 files/build/scripts/set_build_hooks diff --git a/.gitignore b/.gitignore index 47674a3f05e..2a2d70646c7 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ platform/broadcom/sonic-platform-modules-dell/z9264f/sonic_platform/ipmihelper.p # buildinfo files/build/buildinfo +files/build/tmp dockers/**/buildinfo platform/**/buildinfo sonic-slave*/**/buildinfo diff --git a/Makefile b/Makefile index 12e760f6dd6..72c66c4cec0 100644 --- a/Makefile +++ b/Makefile @@ -37,4 +37,4 @@ endif # Freeze the versions, see more detail options: scripts/versions_manager.py freeze -h freeze: - @scripts/versions_manager.py freeze $(OPTIONS) + @scripts/versions_manager.py freeze $(FREEZE_VERSION_OPTIONS) diff --git a/Makefile.work b/Makefile.work index 7f4a4d7ada1..15f70003195 100644 --- a/Makefile.work +++ b/Makefile.work @@ -183,8 +183,7 @@ DOCKER_BASE_BUILD = docker build --no-cache \ -t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \ --build-arg http_proxy=$(http_proxy) \ --build-arg https_proxy=$(https_proxy) \ - $(SLAVE_DIR); \ - scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target + $(SLAVE_DIR) DOCKER_BUILD = docker build --no-cache \ --build-arg user=$(USER) \ @@ -247,9 +246,12 @@ endif endif @$(OVERLAY_MODULE_CHECK) + @pushd files/build; make all; popd + @cp files/build/buildinfo/sonic-build-tools* $(SLAVE_BASE_IMAGE)/buildinfo @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \ - $(DOCKER_BASE_BUILD) ; } + $(DOCKER_BASE_BUILD) ; \ + scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target ; } @docker inspect --type image $(SLAVE_IMAGE):$(SLAVE_TAG) &> /dev/null || \ { echo Image $(SLAVE_IMAGE):$(SLAVE_TAG) not found. Building... ; \ $(DOCKER_BUILD) ; } @@ -263,11 +265,16 @@ else @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?" endif -sonic-slave-base-build : +sonic-build-tools: + @pushd files/build; make all; popd + @cp files/build/buildinfo/sonic-build-tools* $(SLAVE_BASE_IMAGE)/buildinfo + +sonic-slave-base-build : sonic-build-tools @$(OVERLAY_MODULE_CHECK) @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \ - $(DOCKER_BASE_BUILD) ; } + $(DOCKER_BASE_BUILD) ; \ + scripts/collect_docker_version_files.sh $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) target ; } sonic-slave-build : sonic-slave-base-build @docker inspect --type image $(SLAVE_IMAGE):$(SLAVE_TAG) &> /dev/null || \ diff --git a/files/build/Makefile b/files/build/Makefile new file mode 100644 index 00000000000..f4358b871ed --- /dev/null +++ b/files/build/Makefile @@ -0,0 +1,37 @@ +SONIC_BUILD_TOOLS = sonic-build-tools +SONIC_BUILD_TOOLS_VERSION = 1.0 +SONIC_BUILD_TOOLS_PACKAGE = $(SONIC_BUILD_TOOLS)_$(SONIC_BUILD_TOOLS_VERSION)_all.deb + +BUILDINFO_DIR = buildinfo +TMP_DIR = tmp +SYMBOL_LINKS_SRC_DIR = ../../usr/local/share/buildinfo/scripts +SYMBOL_LINKS = set_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files +SONIC_BUILD_TOOLS_TARGET = $(BUILDINFO_DIR)/$(SONIC_BUILD_TOOLS_PACKAGE) +BUILD_ROOT_DIR = $(TMP_DIR)/$(SONIC_BUILD_TOOLS) +DEBIAN_DIR = $(BUILD_ROOT_DIR)/DEBIAN +INSTALL_PATH = $(BUILD_ROOT_DIR)/usr/local/share/buildinfo +SYMBOL_LINK_PATH = $(BUILD_ROOT_DIR)/usr/sbin +SCRIPTS_PATH = $(INSTALL_PATH)/scripts +HOOKS_PATH = $(INSTALL_PATH)/hooks +DPKGTOOL = $(shell which dpkg-deb) + + +# If the depk-deb not installed, use the docker container to make the debian package +ifeq ($(shell which dpkg-deb),) +BUILD_COMMAND=docker run --rm -v $(shell pwd):/build debian bash -c "cd build; dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_TOOLS) $(SONIC_BUILD_TOOLS_TARGET)" +else +BUILD_COMMAND=dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_TOOLS) $(SONIC_BUILD_TOOLS_TARGET) +endif + +DEPENDS := $(shell find scripts hooks debian -type f) +$(SONIC_BUILD_TOOLS_TARGET): $(DEPENDS) + @rm -rf $(BUILDINFO_DIR)/$(SONIC_BUILD_TOOLS) $(TMP_DIR) + @mkdir -p $(DEBIAN_DIR) $(SCRIPTS_PATH) $(HOOKS_PATH) $(SYMBOL_LINK_PATH) + @cp debian/* $(DEBIAN_DIR)/ + @cp scripts/* $(SCRIPTS_PATH)/ + @cp hooks/* $(HOOKS_PATH)/ + @for f in $(SYMBOL_LINKS); do ln -s $(SYMBOL_LINKS_SRC_DIR)/$$f $(SYMBOL_LINK_PATH)/$$f; done + @$(BUILD_COMMAND) + @sudo chmod a+rw $(SONIC_BUILD_TOOLS_TARGET) + +all: $(SONIC_BUILD_TOOLS_TARGET) diff --git a/files/build/debian/control b/files/build/debian/control new file mode 100644 index 00000000000..00de85d1107 --- /dev/null +++ b/files/build/debian/control @@ -0,0 +1,10 @@ +Package: sonic-build-tools +Version: 1.0 +Section: devel +Priority: optional +Architecture: all +Depends: +Maintainer: SONiC +Description: sonic build tools + Hooks the build tools, such as apt-get, wget, pip, etc. + It is used to monitor and control the packages installed during the build. diff --git a/files/build/scripts/apt-get b/files/build/hooks/apt-get similarity index 100% rename from files/build/scripts/apt-get rename to files/build/hooks/apt-get diff --git a/files/build/scripts/curl b/files/build/hooks/curl similarity index 100% rename from files/build/scripts/curl rename to files/build/hooks/curl diff --git a/files/build/scripts/pip b/files/build/hooks/pip similarity index 100% rename from files/build/scripts/pip rename to files/build/hooks/pip diff --git a/files/build/scripts/pip2 b/files/build/hooks/pip2 similarity index 100% rename from files/build/scripts/pip2 rename to files/build/hooks/pip2 diff --git a/files/build/scripts/pip3 b/files/build/hooks/pip3 similarity index 100% rename from files/build/scripts/pip3 rename to files/build/hooks/pip3 diff --git a/files/build/scripts/wget b/files/build/hooks/wget similarity index 100% rename from files/build/scripts/wget rename to files/build/hooks/wget diff --git a/files/build/scripts/buildinfo_base.sh b/files/build/scripts/buildinfo_base.sh index 7f9de6e436c..d19558c544e 100755 --- a/files/build/scripts/buildinfo_base.sh +++ b/files/build/scripts/buildinfo_base.sh @@ -24,7 +24,7 @@ log_err() get_command() { - local path=$(echo $PATH | sed 's#[^:]*buildinfo/scripts:##') + local path=$(echo $PATH | sed 's#[^:]*buildinfo/scripts:##' | sed "s#/usr/sbin:##") local command=$(PATH=$path which $1) echo $command } diff --git a/files/build/scripts/pre_run_buildinfo b/files/build/scripts/pre_run_buildinfo index 961277f7545..6d9f7044fde 100755 --- a/files/build/scripts/pre_run_buildinfo +++ b/files/build/scripts/pre_run_buildinfo @@ -9,6 +9,7 @@ mkdir -p $LOG_PATH [ -d $PRE_VERSION_PATH ] && rm -rf $PRE_VERSION_PATH collect_version_files $PRE_VERSION_PATH +set_build_hooks chmod -R a+rw $BUILDINFO_PATH diff --git a/files/build/scripts/set_build_hooks b/files/build/scripts/set_build_hooks new file mode 100755 index 00000000000..c8c1ee59d72 --- /dev/null +++ b/files/build/scripts/set_build_hooks @@ -0,0 +1,32 @@ +#!/bin/bash + +HOOK_PATH=/usr/local/share/buildinfo/hooks +TARGET_PATH=/usr/sbin +FILES=$(ls $HOOK_PATH) + +usage() +{ + echo "Usage: $0 [-d]" + exit 1 +} + +DISABLE=n +while getopts "d" opt; do + case $opt in + d) + DISABLE=y + ;; + *) + usage + ;; + esac +done + +for f in $FILES +do + if [ $DISABLE == "n" ]; then + [ ! -e $TARGET_PATH/$f ] && ln -s $HOOK_PATH/$f $TARGET_PATH/$f + else + ([ -e $TARGET_PATH/$f ] && ls -l $TARGET_PATH/$f | grep -q $HOOK_PATH) && rm -f $TARGET_PATH/$f + fi +done diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh index 4523ef84d5e..75d4fbad9a8 100755 --- a/scripts/collect_docker_version_files.sh +++ b/scripts/collect_docker_version_files.sh @@ -13,7 +13,12 @@ TARGET_VERSIONS_PATH=$TARGET_PATH/versions/dockers/$DOCKER_IMAGE_NAME mkdir -p $TARGET_VERSIONS_PATH export DOCKER_CLI_EXPERIMENTAL=enabled -docker run --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE > /dev/null 2>&1 + +# Remove the old docker container if existing +if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then + docker container rm $DOCKER_IMAGE > /dev/null 2>&1 +fi +docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE > /dev/null 2>&1 docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/diff-versions $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 mv $TARGET_VERSIONS_PATH/diff-versions/* $TARGET_VERSIONS_PATH/ diff --git a/scripts/prepare_base_image_buildinfo.sh b/scripts/prepare_base_image_buildinfo.sh index 588827c7f86..e7301f462da 100755 --- a/scripts/prepare_base_image_buildinfo.sh +++ b/scripts/prepare_base_image_buildinfo.sh @@ -32,4 +32,5 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ]; then cp -f $BUILDINFO_VERSION_DEB ${FILESYSTEM_ROOT}/etc/apt/preferences.d/ fi +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-tools_1.0_all.deb" sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo" diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 5050d5cd8c9..5bce65d9cda 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -24,8 +24,7 @@ fi DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo COPY ["buildinfo", "/usr/local/share/buildinfo"] -ENV OLDPATH=$PATH -ENV PATH="/usr/local/share/buildinfo/scripts:$PATH" +RUN dpkg -i /usr/local/share/buildinfo/sonic-build-tools_1.0_all.deb RUN pre_run_buildinfo' # Add the auto-generate code if it is not added in the target Dockerfile @@ -37,8 +36,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE # Append the docker build script at the end of the docker file echo "RUN post_run_buildinfo" >> $TEMP_FILE - [ -z "$BUILD_SLAVE" ] && BUILD_SLAVE=n - [ "$BUILD_SLAVE" != "y" ] && echo "ENV PATH=\$OLDPATH" >> $TEMP_FILE + [ "$BUILD_SLAVE" != "y" ] && echo "RUN set_build_hooks -d" >> $TEMP_FILE cat $TEMP_FILE > $DOCKERFILE_TARGE rm -f $TEMP_FILE @@ -47,9 +45,6 @@ fi # Copy the build info config cp -rf files/build/buildinfo/* $BUILDINFO_PATH -# Copy the docker build info scirpts -cp -r files/build/scripts "${BUILDINFO_PATH}/" - # Build the slave running config if [ "$BUILD_SLAVE" == "y" ]; then scripts/versions_manager.py generate -t "${BUILDINFO_PATH}/build/versions" -n "build-${IMAGENAME}" -d "$DISTRO" -a "$ARCH" From aafeb8b02ee484e896d778c3e44ce90448aeaa33 Mon Sep 17 00:00:00 2001 From: xumia Date: Sun, 6 Dec 2020 05:58:20 +0000 Subject: [PATCH 10/18] Fix auto debian package removed issue --- scripts/build_debian_base_system.sh | 4 ++-- scripts/prepare_base_image_buildinfo.sh | 7 ++----- scripts/prepare_docker_buildinfo.sh | 1 + 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/build_debian_base_system.sh b/scripts/build_debian_base_system.sh index e1539df05db..a85b3b5bf0e 100755 --- a/scripts/build_debian_base_system.sh +++ b/scripts/build_debian_base_system.sh @@ -23,9 +23,9 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ]; then # qemu arm bin executable for cross-building sudo mkdir -p $FILESYSTEM_ROOT/usr/bin sudo cp /usr/bin/qemu*static $FILESYSTEM_ROOT/usr/bin || true - sudo http_proxy=$HTTP_PROXY debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian + sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://deb.debian.org/debian else - sudo http_proxy=$HTTP_PROXY debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian + sudo http_proxy=$HTTP_PROXY SKIP_BUILD_HOOK=y debootstrap --variant=minbase --arch $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT http://debian-archive.trafficmanager.net/debian fi RET=$? if [ $RET -ne 0 ]; then diff --git a/scripts/prepare_base_image_buildinfo.sh b/scripts/prepare_base_image_buildinfo.sh index e7301f462da..729f2a856e2 100755 --- a/scripts/prepare_base_image_buildinfo.sh +++ b/scripts/prepare_base_image_buildinfo.sh @@ -8,29 +8,26 @@ FILESYSTEM_ROOT=$3 . /usr/local/share/buildinfo/scripts/buildinfo_base.sh VERSION_DEB_PREFERENCE="01-versions-deb" BUILDINFO_PATH=${FILESYSTEM_ROOT}/usr/local/share/buildinfo -BUILDINFO_INSTALL_PATH=${FILESYSTEM_ROOT}/usr/local/sbin BUILDINFO_VERSION_PATH=${FILESYSTEM_ROOT}/usr/local/share/buildinfo/versions BUILDINFO_VERSION_DEB=${BUILDINFO_VERSION_PATH}/${VERSION_DEB_PREFERENCE} OVERRIDE_VERSION_PATH=files/build/versions/host-image DIFF_VERSIONS_PATH=$BUILDINFO_PATH/diff-versions -# Copy build info scripts mkdir -p $BUILDINFO_PATH -cp -rf files/build/scripts ${BUILDINFO_PATH}/ # Copy the build info config cp -rf files/build/buildinfo/* $BUILDINFO_PATH/ # Install the config files -cp -rf $BUILDINFO_PATH/scripts/* "$BUILDINFO_INSTALL_PATH/" cp $BUILDINFO_PATH/trusted.gpg.d/* "${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/" # Generate version lock files scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -m "$OVERRIDE_VERSION_PATH" -d "$DISTRO" -a "$ARCH" -if [ "$ENABLE_VERSION_CONTROL_DEB" != "y" ]; then +if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then cp -f $BUILDINFO_VERSION_DEB ${FILESYSTEM_ROOT}/etc/apt/preferences.d/ fi sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-tools_1.0_all.deb" +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "apt-mark hold sonic-build-tools" sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo" diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 5bce65d9cda..85dd36b57e9 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -25,6 +25,7 @@ fi DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo COPY ["buildinfo", "/usr/local/share/buildinfo"] RUN dpkg -i /usr/local/share/buildinfo/sonic-build-tools_1.0_all.deb +RUN apt-mark hold sonic-build-tools RUN pre_run_buildinfo' # Add the auto-generate code if it is not added in the target Dockerfile From 47d8ae05638f258fb941b4c79390dc252e662488 Mon Sep 17 00:00:00 2001 From: xumia Date: Tue, 8 Dec 2020 02:39:57 +0000 Subject: [PATCH 11/18] Change build debian package name, and change the folder --- Makefile.work | 12 +++--- scripts/collect_docker_version_files.sh | 4 +- scripts/generate_buildinfo_config.sh | 2 +- scripts/prepare_base_image_buildinfo.sh | 6 +-- scripts/prepare_docker_buildinfo.sh | 6 +-- scripts/versions_manager.py | 40 +++++++++++++------ src/sonic-build-hooks/.gitignore | 2 + .../build => src/sonic-build-hooks}/Makefile | 22 +++++----- .../sonic-build-hooks}/debian/control | 4 +- .../sonic-build-hooks}/hooks/apt-get | 0 .../sonic-build-hooks}/hooks/curl | 0 .../build => src/sonic-build-hooks}/hooks/pip | 0 .../sonic-build-hooks}/hooks/pip2 | 0 .../sonic-build-hooks}/hooks/pip3 | 0 .../sonic-build-hooks}/hooks/wget | 0 .../scripts/buildinfo_base.sh | 0 .../scripts/collect_version_files | 0 .../scripts/post_run_buildinfo | 0 .../scripts/pre_run_buildinfo | 0 .../scripts/set_build_hooks | 0 20 files changed, 58 insertions(+), 40 deletions(-) create mode 100644 src/sonic-build-hooks/.gitignore rename {files/build => src/sonic-build-hooks}/Makefile (60%) rename {files/build => src/sonic-build-hooks}/debian/control (81%) rename {files/build => src/sonic-build-hooks}/hooks/apt-get (100%) rename {files/build => src/sonic-build-hooks}/hooks/curl (100%) rename {files/build => src/sonic-build-hooks}/hooks/pip (100%) rename {files/build => src/sonic-build-hooks}/hooks/pip2 (100%) rename {files/build => src/sonic-build-hooks}/hooks/pip3 (100%) rename {files/build => src/sonic-build-hooks}/hooks/wget (100%) rename {files/build => src/sonic-build-hooks}/scripts/buildinfo_base.sh (100%) rename {files/build => src/sonic-build-hooks}/scripts/collect_version_files (100%) rename {files/build => src/sonic-build-hooks}/scripts/post_run_buildinfo (100%) rename {files/build => src/sonic-build-hooks}/scripts/pre_run_buildinfo (100%) rename {files/build => src/sonic-build-hooks}/scripts/set_build_hooks (100%) diff --git a/Makefile.work b/Makefile.work index 15f70003195..54f5e4778de 100644 --- a/Makefile.work +++ b/Makefile.work @@ -246,8 +246,8 @@ endif endif @$(OVERLAY_MODULE_CHECK) - @pushd files/build; make all; popd - @cp files/build/buildinfo/sonic-build-tools* $(SLAVE_BASE_IMAGE)/buildinfo + @pushd src/sonic-build-hooks; make all; popd + @cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_BASE_IMAGE)/buildinfo @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \ $(DOCKER_BASE_BUILD) ; \ @@ -265,11 +265,11 @@ else @$(DOCKER_RUN) $(SLAVE_IMAGE):$(SLAVE_TAG) bash -c "$(SONIC_BUILD_INSTRUCTION) $@; scripts/collect_build_version_files.sh \$$?" endif -sonic-build-tools: - @pushd files/build; make all; popd - @cp files/build/buildinfo/sonic-build-tools* $(SLAVE_BASE_IMAGE)/buildinfo +sonic-build-hooks: + @pushd src/sonic-build-hooks; make all; popd + @cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_BASE_IMAGE)/buildinfo -sonic-slave-base-build : sonic-build-tools +sonic-slave-base-build : sonic-build-hooks @$(OVERLAY_MODULE_CHECK) @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \ diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh index 75d4fbad9a8..28530447105 100755 --- a/scripts/collect_docker_version_files.sh +++ b/scripts/collect_docker_version_files.sh @@ -16,9 +16,9 @@ export DOCKER_CLI_EXPERIMENTAL=enabled # Remove the old docker container if existing if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then - docker container rm $DOCKER_IMAGE > /dev/null 2>&1 + docker container rm $DOCKER_IMAGE > /dev/null fi -docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE > /dev/null 2>&1 +docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/diff-versions $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 mv $TARGET_VERSIONS_PATH/diff-versions/* $TARGET_VERSIONS_PATH/ diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh index 5e30addf5de..63429258cef 100755 --- a/scripts/generate_buildinfo_config.sh +++ b/scripts/generate_buildinfo_config.sh @@ -1,6 +1,6 @@ #!/bin/bash -BUILDINFO_PATH=files/build +BUILDINFO_PATH=src/sonic-build-hooks TRUSTED_GPG_PATH=$BUILDINFO_PATH/buildinfo/trusted.gpg.d BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config diff --git a/scripts/prepare_base_image_buildinfo.sh b/scripts/prepare_base_image_buildinfo.sh index 729f2a856e2..c3a9dab39cd 100755 --- a/scripts/prepare_base_image_buildinfo.sh +++ b/scripts/prepare_base_image_buildinfo.sh @@ -16,7 +16,7 @@ DIFF_VERSIONS_PATH=$BUILDINFO_PATH/diff-versions mkdir -p $BUILDINFO_PATH # Copy the build info config -cp -rf files/build/buildinfo/* $BUILDINFO_PATH/ +cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH/ # Install the config files cp $BUILDINFO_PATH/trusted.gpg.d/* "${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/" @@ -28,6 +28,6 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then cp -f $BUILDINFO_VERSION_DEB ${FILESYSTEM_ROOT}/etc/apt/preferences.d/ fi -sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-tools_1.0_all.deb" -sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "apt-mark hold sonic-build-tools" +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb" +sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "apt-mark hold sonic-build-hooks" sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo" diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 85dd36b57e9..8b88e7864af 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -24,8 +24,8 @@ fi DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo COPY ["buildinfo", "/usr/local/share/buildinfo"] -RUN dpkg -i /usr/local/share/buildinfo/sonic-build-tools_1.0_all.deb -RUN apt-mark hold sonic-build-tools +RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb +RUN apt-mark hold sonic-build-hooks RUN pre_run_buildinfo' # Add the auto-generate code if it is not added in the target Dockerfile @@ -44,7 +44,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE fi # Copy the build info config -cp -rf files/build/buildinfo/* $BUILDINFO_PATH +cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH # Build the slave running config if [ "$BUILD_SLAVE" == "y" ]; then diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py index 4dc4c51308b..08d20bd9a93 100755 --- a/scripts/versions_manager.py +++ b/scripts/versions_manager.py @@ -175,11 +175,17 @@ def overwrite(self, module, for_all_dist=False, for_all_arch=False): def get_config_module(self, default_module, dist, arch): if self.is_individule_version(): return self + module = default_module + if not self.is_aggregatable_module(self.name): + module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) + return self._get_config_module(module, dist, arch) + + def _get_config_module(self, default_module, dist, arch): module = default_module.clone() + default_ctype_components = module._get_components_per_ctypes() module.overwrite(self) config_components = [] ctype_components = module._get_components_per_ctypes() - default_ctype_components = default_module._get_components_per_ctypes() for ctype in default_ctype_components: if ctype not in ctype_components: ctype_components[ctype] = [] @@ -206,7 +212,7 @@ def subtract(self, default_module): for i in range(0, len(components)): component = components[i] base_module = VersionModule(self.name, components[0:i]) - config_module = base_module.get_config_module(default_module, component.dist, component.arch) + config_module = base_module._get_config_module(default_module, component.dist, component.arch) config_components = config_module._get_components_by_ctype(ctype) if len(config_components) > 0: config_component = config_components[0] @@ -320,6 +326,18 @@ def is_slave_module(self): def is_individule_version(self): return self.is_slave_module() and SLAVE_INDIVIDULE_VERSION + @classmethod + def is_aggregatable_module(cls, module_name): + if module_name.startswith('sonic-slave-'): + return False + if module_name.startswith('build-sonic-slave-'): + return False + if module_name == DEFAULT_MODULE: + return False + if module_name == 'host-image' or module_name == 'host-base-image': + return False + return True + @classmethod def get_module_path_by_name(cls, source_path, module_name): common_modules = ['default', 'host-image', 'host-base-image'] @@ -388,6 +406,7 @@ def dump(self): module.dump(module_path) def subtract(self, default_module): + none_aggregatable_module = default_module.clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) for module in self.modules.values(): if module.name == DEFAULT_MODULE: continue @@ -395,7 +414,10 @@ def subtract(self, default_module): continue if module.is_individule_version(): continue - module.subtract(default_module) + tmp_module = default_module + if not module.is_aggregatable_module(module.name): + tmp_module = none_aggregatable_module + module.subtract(tmp_module) def freeze(self, rebuild=False, for_all_dist=False, for_all_arch=False, ctypes=['all']): if rebuild: @@ -447,16 +469,10 @@ def get_default_module(self): components.append(component) return VersionModule(DEFAULT_MODULE, components) - def get_docker_version_modules(self): + def get_aggregatable_modules(self): modules = {} for module_name in self.modules: - if module_name.startswith('sonic-slave-'): - continue - if module_name.startswith('build-sonic-slave-'): - continue - if module_name == DEFAULT_MODULE: - continue - if module_name == 'host-image' or module_name == 'host-base-image': + if not VersionModule.is_aggregatable_module(module_name): continue module = self.modules[module_name] modules[module_name] = module @@ -522,7 +538,7 @@ def _clean_component_info(self, clean_dist=True, clean_arch=True): def _get_versions(self, ctype, dist=None, arch=None): versions = {} - modules = self.get_docker_version_modules() + modules = self.get_aggregatable_modules() for module_name in self.modules: if module_name not in modules: temp_module = self.modules[module_name].clone(exclude_ctypes=DEFAULT_OVERWRITE_COMPONENTS) diff --git a/src/sonic-build-hooks/.gitignore b/src/sonic-build-hooks/.gitignore new file mode 100644 index 00000000000..4490afae4a5 --- /dev/null +++ b/src/sonic-build-hooks/.gitignore @@ -0,0 +1,2 @@ +buildinfo +tmp diff --git a/files/build/Makefile b/src/sonic-build-hooks/Makefile similarity index 60% rename from files/build/Makefile rename to src/sonic-build-hooks/Makefile index f4358b871ed..0695c42c0a3 100644 --- a/files/build/Makefile +++ b/src/sonic-build-hooks/Makefile @@ -1,13 +1,13 @@ -SONIC_BUILD_TOOLS = sonic-build-tools -SONIC_BUILD_TOOLS_VERSION = 1.0 -SONIC_BUILD_TOOLS_PACKAGE = $(SONIC_BUILD_TOOLS)_$(SONIC_BUILD_TOOLS_VERSION)_all.deb +SONIC_BUILD_HOOKS = sonic-build-hooks +SONIC_BUILD_HOOKS_VERSION = 1.0 +SONIC_BUILD_HOOKS_PACKAGE = $(SONIC_BUILD_HOOKS)_$(SONIC_BUILD_HOOKS_VERSION)_all.deb BUILDINFO_DIR = buildinfo TMP_DIR = tmp SYMBOL_LINKS_SRC_DIR = ../../usr/local/share/buildinfo/scripts SYMBOL_LINKS = set_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files -SONIC_BUILD_TOOLS_TARGET = $(BUILDINFO_DIR)/$(SONIC_BUILD_TOOLS_PACKAGE) -BUILD_ROOT_DIR = $(TMP_DIR)/$(SONIC_BUILD_TOOLS) +SONIC_BUILD_HOOKS_TARGET = $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS_PACKAGE) +BUILD_ROOT_DIR = $(TMP_DIR)/$(SONIC_BUILD_HOOKS) DEBIAN_DIR = $(BUILD_ROOT_DIR)/DEBIAN INSTALL_PATH = $(BUILD_ROOT_DIR)/usr/local/share/buildinfo SYMBOL_LINK_PATH = $(BUILD_ROOT_DIR)/usr/sbin @@ -18,20 +18,20 @@ DPKGTOOL = $(shell which dpkg-deb) # If the depk-deb not installed, use the docker container to make the debian package ifeq ($(shell which dpkg-deb),) -BUILD_COMMAND=docker run --rm -v $(shell pwd):/build debian bash -c "cd build; dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_TOOLS) $(SONIC_BUILD_TOOLS_TARGET)" +BUILD_COMMAND=docker run --rm -v $(shell pwd):/build debian bash -c "cd build; dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_HOOKS) $(SONIC_BUILD_HOOKS_TARGET)" else -BUILD_COMMAND=dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_TOOLS) $(SONIC_BUILD_TOOLS_TARGET) +BUILD_COMMAND=dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_HOOKS) $(SONIC_BUILD_HOOKS_TARGET) endif DEPENDS := $(shell find scripts hooks debian -type f) -$(SONIC_BUILD_TOOLS_TARGET): $(DEPENDS) - @rm -rf $(BUILDINFO_DIR)/$(SONIC_BUILD_TOOLS) $(TMP_DIR) +$(SONIC_BUILD_HOOKS_TARGET): $(DEPENDS) + @rm -rf $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS) $(TMP_DIR) @mkdir -p $(DEBIAN_DIR) $(SCRIPTS_PATH) $(HOOKS_PATH) $(SYMBOL_LINK_PATH) @cp debian/* $(DEBIAN_DIR)/ @cp scripts/* $(SCRIPTS_PATH)/ @cp hooks/* $(HOOKS_PATH)/ @for f in $(SYMBOL_LINKS); do ln -s $(SYMBOL_LINKS_SRC_DIR)/$$f $(SYMBOL_LINK_PATH)/$$f; done @$(BUILD_COMMAND) - @sudo chmod a+rw $(SONIC_BUILD_TOOLS_TARGET) + @sudo chmod a+rw $(SONIC_BUILD_HOOKS_TARGET) -all: $(SONIC_BUILD_TOOLS_TARGET) +all: $(SONIC_BUILD_HOOKS_TARGET) diff --git a/files/build/debian/control b/src/sonic-build-hooks/debian/control similarity index 81% rename from files/build/debian/control rename to src/sonic-build-hooks/debian/control index 00de85d1107..9e2454be9d9 100644 --- a/files/build/debian/control +++ b/src/sonic-build-hooks/debian/control @@ -1,10 +1,10 @@ -Package: sonic-build-tools +Package: sonic-build-hooks Version: 1.0 Section: devel Priority: optional Architecture: all Depends: Maintainer: SONiC -Description: sonic build tools +Description: sonic build hooks Hooks the build tools, such as apt-get, wget, pip, etc. It is used to monitor and control the packages installed during the build. diff --git a/files/build/hooks/apt-get b/src/sonic-build-hooks/hooks/apt-get similarity index 100% rename from files/build/hooks/apt-get rename to src/sonic-build-hooks/hooks/apt-get diff --git a/files/build/hooks/curl b/src/sonic-build-hooks/hooks/curl similarity index 100% rename from files/build/hooks/curl rename to src/sonic-build-hooks/hooks/curl diff --git a/files/build/hooks/pip b/src/sonic-build-hooks/hooks/pip similarity index 100% rename from files/build/hooks/pip rename to src/sonic-build-hooks/hooks/pip diff --git a/files/build/hooks/pip2 b/src/sonic-build-hooks/hooks/pip2 similarity index 100% rename from files/build/hooks/pip2 rename to src/sonic-build-hooks/hooks/pip2 diff --git a/files/build/hooks/pip3 b/src/sonic-build-hooks/hooks/pip3 similarity index 100% rename from files/build/hooks/pip3 rename to src/sonic-build-hooks/hooks/pip3 diff --git a/files/build/hooks/wget b/src/sonic-build-hooks/hooks/wget similarity index 100% rename from files/build/hooks/wget rename to src/sonic-build-hooks/hooks/wget diff --git a/files/build/scripts/buildinfo_base.sh b/src/sonic-build-hooks/scripts/buildinfo_base.sh similarity index 100% rename from files/build/scripts/buildinfo_base.sh rename to src/sonic-build-hooks/scripts/buildinfo_base.sh diff --git a/files/build/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files similarity index 100% rename from files/build/scripts/collect_version_files rename to src/sonic-build-hooks/scripts/collect_version_files diff --git a/files/build/scripts/post_run_buildinfo b/src/sonic-build-hooks/scripts/post_run_buildinfo similarity index 100% rename from files/build/scripts/post_run_buildinfo rename to src/sonic-build-hooks/scripts/post_run_buildinfo diff --git a/files/build/scripts/pre_run_buildinfo b/src/sonic-build-hooks/scripts/pre_run_buildinfo similarity index 100% rename from files/build/scripts/pre_run_buildinfo rename to src/sonic-build-hooks/scripts/pre_run_buildinfo diff --git a/files/build/scripts/set_build_hooks b/src/sonic-build-hooks/scripts/set_build_hooks similarity index 100% rename from files/build/scripts/set_build_hooks rename to src/sonic-build-hooks/scripts/set_build_hooks From 335fc1452e0dfe5ce2dae3e63ceb1feb79467126 Mon Sep 17 00:00:00 2001 From: xumia Date: Wed, 9 Dec 2020 03:16:26 +0000 Subject: [PATCH 12/18] Collect the pre-versions and post-versions --- build_debian.sh | 8 +++++++- scripts/collect_docker_version_files.sh | 5 ++--- scripts/collect_host_image_version_files.sh | 3 ++- ...nfo.sh => prepare_debian_image_buildinfo.sh} | 3 --- scripts/prepare_docker_buildinfo.sh | 3 +-- scripts/prepare_slave_container_buildinfo.sh | 3 +++ scripts/versions_manager.py | 12 +++++++++++- .../scripts/collect_version_files | 1 + .../scripts/post_run_buildinfo | 17 +++-------------- src/sonic-build-hooks/scripts/pre_run_buildinfo | 2 -- src/sonic-build-hooks/scripts/set_build_hooks | 2 ++ 11 files changed, 32 insertions(+), 27 deletions(-) rename scripts/{prepare_base_image_buildinfo.sh => prepare_debian_image_buildinfo.sh} (91%) diff --git a/build_debian.sh b/build_debian.sh index b168fb76e21..e471ce967fb 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -40,6 +40,8 @@ PLATFORM_DIR=platform ## Hostname for the linux image HOSTNAME=sonic DEFAULT_USERINFO="Default admin user,,," +BUILD_TOOL_PATH=src/sonic-build-hooks/buildinfo +TRUSTED_GPG_DIR=$BUILD_TOOL_PATH/trusted.gpg.d ## Read ONIE image related config file . ./onie-image.conf @@ -75,7 +77,7 @@ echo '[INFO] Build host debian base system...' TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT # Prepare buildinfo -sudo scripts/prepare_base_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy +sudo scripts/prepare_debian_image_buildinfo.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT $http_proxy ## Config hostname and hosts, otherwise 'sudo ...' will complain 'sudo: unable to resolve host ...' sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "echo '$HOSTNAME' > /etc/hostname" @@ -96,6 +98,9 @@ echo '[INFO] Mount all' ## Output all the mounted device for troubleshooting sudo LANG=C chroot $FILESYSTEM_ROOT mount +## Install the trusted gpg public keys +[ -d $TRUSTED_GPG_DIR ] && [ ! -z "$(ls $TRUSTED_GPG_DIR)" ] && sudo cp $TRUSTED_GPG_DIR/* ${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/ + ## Pointing apt to public apt mirrors and getting latest packages, needed for latest security updates sudo cp files/apt/sources.list.$CONFIGURED_ARCH $FILESYSTEM_ROOT/etc/apt/sources.list sudo cp files/apt/apt.conf.d/{81norecommends,apt-{clean,gzip-indexes,no-languages},no-check-valid-until} $FILESYSTEM_ROOT/etc/apt/apt.conf.d/ @@ -568,6 +573,7 @@ sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT +sudo LANG=C chroot $FILESYSTEM_ROOT set_build_hooks -d ## Compress docker files pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh index 28530447105..7ff03f49ff8 100755 --- a/scripts/collect_docker_version_files.sh +++ b/scripts/collect_docker_version_files.sh @@ -20,7 +20,6 @@ if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then fi docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 -docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/diff-versions $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 -mv $TARGET_VERSIONS_PATH/diff-versions/* $TARGET_VERSIONS_PATH/ -rm -rf $TARGET_VERSIONS_PATH/diff-versions +docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 +docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 docker container rm $DOCKER_CONTAINER > /dev/null 2>&1 diff --git a/scripts/collect_host_image_version_files.sh b/scripts/collect_host_image_version_files.sh index 60d64a9195d..3e06bcfb234 100755 --- a/scripts/collect_host_image_version_files.sh +++ b/scripts/collect_host_image_version_files.sh @@ -8,4 +8,5 @@ mkdir -p $VERSIONS_PATH sudo LANG=C chroot $FILESYSTEM_ROOT post_run_buildinfo -cp $FILESYSTEM_ROOT/usr/local/share/buildinfo/diff-versions/* $VERSIONS_PATH/ +cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/pre-versions $VERSIONS_PATH/ +cp -r $FILESYSTEM_ROOT/usr/local/share/buildinfo/post-versions $VERSIONS_PATH/ diff --git a/scripts/prepare_base_image_buildinfo.sh b/scripts/prepare_debian_image_buildinfo.sh similarity index 91% rename from scripts/prepare_base_image_buildinfo.sh rename to scripts/prepare_debian_image_buildinfo.sh index c3a9dab39cd..42c1833b6bb 100755 --- a/scripts/prepare_base_image_buildinfo.sh +++ b/scripts/prepare_debian_image_buildinfo.sh @@ -18,9 +18,6 @@ mkdir -p $BUILDINFO_PATH # Copy the build info config cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH/ -# Install the config files -cp $BUILDINFO_PATH/trusted.gpg.d/* "${FILESYSTEM_ROOT}/etc/apt/trusted.gpg.d/" - # Generate version lock files scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -m "$OVERRIDE_VERSION_PATH" -d "$DISTRO" -a "$ARCH" diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 8b88e7864af..b5745285dba 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -25,7 +25,7 @@ fi DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo COPY ["buildinfo", "/usr/local/share/buildinfo"] RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb -RUN apt-mark hold sonic-build-hooks +RUN cp -rf /usr/local/share/buildinfo/trusted.gpg.d/* /etc/apt/trusted.gpg.d/ RUN pre_run_buildinfo' # Add the auto-generate code if it is not added in the target Dockerfile @@ -37,7 +37,6 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE # Append the docker build script at the end of the docker file echo "RUN post_run_buildinfo" >> $TEMP_FILE - [ "$BUILD_SLAVE" != "y" ] && echo "RUN set_build_hooks -d" >> $TEMP_FILE cat $TEMP_FILE > $DOCKERFILE_TARGE rm -f $TEMP_FILE diff --git a/scripts/prepare_slave_container_buildinfo.sh b/scripts/prepare_slave_container_buildinfo.sh index 43c68039d89..584c2995be3 100755 --- a/scripts/prepare_slave_container_buildinfo.sh +++ b/scripts/prepare_slave_container_buildinfo.sh @@ -6,6 +6,9 @@ BUILDINFO=$SLAVE_DIR/buildinfo BUILD_VERSIONS_PATH=$SLAVE_DIR/buildinfo/build/versions VERSION_DEB_PREFERENCE=$BUILD_VERSIONS_PATH/01-versions-deb +# Enable the build hooks +set_build_hooks + cp -rf $SLAVE_DIR/buildinfo/* /usr/local/share/buildinfo/ [ -d /usr/local/share/buildinfo/versions ] && rm -rf /usr/local/share/buildinfo/versions diff --git a/scripts/versions_manager.py b/scripts/versions_manager.py index 08d20bd9a93..59621cdfbbc 100755 --- a/scripts/versions_manager.py +++ b/scripts/versions_manager.py @@ -287,7 +287,17 @@ def load(self, image_path, filter_ctype=None, filter_dist=None, filter_arch=None components.append(component) def load_from_target(self, image_path): - self.load(image_path) + post_versions = os.path.join(image_path, 'post-versions') + if os.path.exists(post_versions): + self.load(post_versions) + self.name = os.path.basename(image_path) + pre_versions = os.path.join(image_path, 'pre-versions') + if os.path.exists(pre_versions): + pre_module = VersionModule() + pre_module.load(pre_versions) + self.subtract(pre_module) + else: + self.load(image_path) def dump(self, module_path, config=False, priority=999): version_file_pattern = os.path.join(module_path, VERSION_PREFIX + '*') diff --git a/src/sonic-build-hooks/scripts/collect_version_files b/src/sonic-build-hooks/scripts/collect_version_files index fed1663e783..b62beb4a211 100755 --- a/src/sonic-build-hooks/scripts/collect_version_files +++ b/src/sonic-build-hooks/scripts/collect_version_files @@ -6,6 +6,7 @@ DIST=$(grep VERSION_CODENAME /etc/os-release | cut -d= -f2) ([ -z "$DIST" ] && grep -q jessie /etc/os-release) && DIST=jessie mkdir -p $TARGET_PATH +chmod a+rw $TARGET_PATH dpkg-query -W -f '${Package}==${Version}\n' > "${TARGET_PATH}/versions-deb-${DIST}-${ARCH}" ([ -x "/usr/local/bin/pip2" ] || [ -x "/usr/bin/pip2" ]) && pip2 freeze > "${TARGET_PATH}/versions-py2-${DIST}-${ARCH}" ([ -x "/usr/local/bin/pip3" ] || [ -x "/usr/bin/pip3" ]) && pip3 freeze > "${TARGET_PATH}/versions-py3-${DIST}-${ARCH}" diff --git a/src/sonic-build-hooks/scripts/post_run_buildinfo b/src/sonic-build-hooks/scripts/post_run_buildinfo index 1b2cdbe2124..669d6e8b127 100755 --- a/src/sonic-build-hooks/scripts/post_run_buildinfo +++ b/src/sonic-build-hooks/scripts/post_run_buildinfo @@ -4,19 +4,8 @@ [ -d $POST_VERSION_PATH ] && rm -rf $POST_VERSION_PATH +# Collect the version files collect_version_files $POST_VERSION_PATH -chmod -R a+rw $BUILDINFO_PATH -for file in $POST_VERSION_PATH/versions-* -do - filename=$(basename $file) - init_file=$PRE_VERSION_PATH/$filename - diff_file=$DIFF_VERSION_PATH/$filename - cat $file | sort > $diff_file - if [ -f $init_file ]; then - cat $file $init_file $init_file | sort | LC_ALL=C uniq -u > $diff_file - fi -done - -[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $DIFF_VERSION_PATH/ -rm -rf $BUILD_VERSION_PATH/* +# Disable the build hooks +set_build_hooks -d diff --git a/src/sonic-build-hooks/scripts/pre_run_buildinfo b/src/sonic-build-hooks/scripts/pre_run_buildinfo index 6d9f7044fde..cf77bec7b4e 100755 --- a/src/sonic-build-hooks/scripts/pre_run_buildinfo +++ b/src/sonic-build-hooks/scripts/pre_run_buildinfo @@ -17,6 +17,4 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]; t cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ fi -cp -rf $BUILDINFO_PATH/trusted.gpg.d/* /etc/apt/trusted.gpg.d/ - exit 0 diff --git a/src/sonic-build-hooks/scripts/set_build_hooks b/src/sonic-build-hooks/scripts/set_build_hooks index c8c1ee59d72..7218575ee9a 100755 --- a/src/sonic-build-hooks/scripts/set_build_hooks +++ b/src/sonic-build-hooks/scripts/set_build_hooks @@ -30,3 +30,5 @@ do ([ -e $TARGET_PATH/$f ] && ls -l $TARGET_PATH/$f | grep -q $HOOK_PATH) && rm -f $TARGET_PATH/$f fi done + +exit 0 From 3cc0cf9af91a2109addc820abe023f823ec86c26 Mon Sep 17 00:00:00 2001 From: xumia Date: Wed, 9 Dec 2020 03:27:22 +0000 Subject: [PATCH 13/18] Change to use debian:buster --- src/sonic-build-hooks/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sonic-build-hooks/Makefile b/src/sonic-build-hooks/Makefile index 0695c42c0a3..b30fd65451e 100644 --- a/src/sonic-build-hooks/Makefile +++ b/src/sonic-build-hooks/Makefile @@ -18,7 +18,7 @@ DPKGTOOL = $(shell which dpkg-deb) # If the depk-deb not installed, use the docker container to make the debian package ifeq ($(shell which dpkg-deb),) -BUILD_COMMAND=docker run --rm -v $(shell pwd):/build debian bash -c "cd build; dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_HOOKS) $(SONIC_BUILD_HOOKS_TARGET)" +BUILD_COMMAND=docker run --rm -v $(shell pwd):/build debian:buster bash -c "cd build; dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_HOOKS) $(SONIC_BUILD_HOOKS_TARGET)" else BUILD_COMMAND=dpkg-deb --build $(TMP_DIR)/$(SONIC_BUILD_HOOKS) $(SONIC_BUILD_HOOKS_TARGET) endif From ad49cc9cd12efb6a0e869473239c9bdadca5fe12 Mon Sep 17 00:00:00 2001 From: xumia Date: Thu, 10 Dec 2020 10:20:16 +0000 Subject: [PATCH 14/18] Remove apt-mark and improve code --- scripts/collect_docker_version_files.sh | 8 ++++---- scripts/prepare_debian_image_buildinfo.sh | 1 - scripts/prepare_slave_container_buildinfo.sh | 2 +- src/sonic-build-hooks/Makefile | 2 +- src/sonic-build-hooks/scripts/post_run_buildinfo | 5 ++++- src/sonic-build-hooks/scripts/pre_run_buildinfo | 2 +- .../scripts/{set_build_hooks => symlink_build_hooks} | 0 7 files changed, 11 insertions(+), 9 deletions(-) rename src/sonic-build-hooks/scripts/{set_build_hooks => symlink_build_hooks} (100%) diff --git a/scripts/collect_docker_version_files.sh b/scripts/collect_docker_version_files.sh index 7ff03f49ff8..73f0a9b5319 100755 --- a/scripts/collect_docker_version_files.sh +++ b/scripts/collect_docker_version_files.sh @@ -19,7 +19,7 @@ if docker container inspect $DOCKER_IMAGE > /dev/null 2>&1; then docker container rm $DOCKER_IMAGE > /dev/null fi docker create --name $DOCKER_CONTAINER --entrypoint /bin/bash $DOCKER_IMAGE -docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 -docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 -docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/ > /dev/null 2>&1 -docker container rm $DOCKER_CONTAINER > /dev/null 2>&1 +docker cp -L $DOCKER_CONTAINER:/etc/os-release $TARGET_VERSIONS_PATH/ +docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/pre-versions $TARGET_VERSIONS_PATH/ +docker cp -L $DOCKER_CONTAINER:/usr/local/share/buildinfo/post-versions $TARGET_VERSIONS_PATH/ +docker container rm $DOCKER_CONTAINER diff --git a/scripts/prepare_debian_image_buildinfo.sh b/scripts/prepare_debian_image_buildinfo.sh index 42c1833b6bb..912e0de0b25 100755 --- a/scripts/prepare_debian_image_buildinfo.sh +++ b/scripts/prepare_debian_image_buildinfo.sh @@ -26,5 +26,4 @@ if [ "$ENABLE_VERSION_CONTROL_DEB" == "y" ]; then fi sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb" -sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "apt-mark hold sonic-build-hooks" sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "pre_run_buildinfo" diff --git a/scripts/prepare_slave_container_buildinfo.sh b/scripts/prepare_slave_container_buildinfo.sh index 584c2995be3..8bc9c719a9e 100755 --- a/scripts/prepare_slave_container_buildinfo.sh +++ b/scripts/prepare_slave_container_buildinfo.sh @@ -7,7 +7,7 @@ BUILD_VERSIONS_PATH=$SLAVE_DIR/buildinfo/build/versions VERSION_DEB_PREFERENCE=$BUILD_VERSIONS_PATH/01-versions-deb # Enable the build hooks -set_build_hooks +symlink_build_hooks cp -rf $SLAVE_DIR/buildinfo/* /usr/local/share/buildinfo/ diff --git a/src/sonic-build-hooks/Makefile b/src/sonic-build-hooks/Makefile index b30fd65451e..f0ec282f8de 100644 --- a/src/sonic-build-hooks/Makefile +++ b/src/sonic-build-hooks/Makefile @@ -5,7 +5,7 @@ SONIC_BUILD_HOOKS_PACKAGE = $(SONIC_BUILD_HOOKS)_$(SONIC_BUILD_HOOKS_VERSION)_al BUILDINFO_DIR = buildinfo TMP_DIR = tmp SYMBOL_LINKS_SRC_DIR = ../../usr/local/share/buildinfo/scripts -SYMBOL_LINKS = set_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files +SYMBOL_LINKS = symlink_build_hooks post_run_buildinfo pre_run_buildinfo collect_version_files SONIC_BUILD_HOOKS_TARGET = $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS_PACKAGE) BUILD_ROOT_DIR = $(TMP_DIR)/$(SONIC_BUILD_HOOKS) DEBIAN_DIR = $(BUILD_ROOT_DIR)/DEBIAN diff --git a/src/sonic-build-hooks/scripts/post_run_buildinfo b/src/sonic-build-hooks/scripts/post_run_buildinfo index 669d6e8b127..a8dab41021b 100755 --- a/src/sonic-build-hooks/scripts/post_run_buildinfo +++ b/src/sonic-build-hooks/scripts/post_run_buildinfo @@ -7,5 +7,8 @@ # Collect the version files collect_version_files $POST_VERSION_PATH +[ -d $BUILD_VERSION_PATH ] && [ ! -z "$(ls -A $BUILD_VERSION_PATH)" ] && cp -rf $BUILD_VERSION_PATH/* $POST_VERSION_PATH +rm -rf $BUILD_VERSION_PATH/* + # Disable the build hooks -set_build_hooks -d +symlink_build_hooks -d diff --git a/src/sonic-build-hooks/scripts/pre_run_buildinfo b/src/sonic-build-hooks/scripts/pre_run_buildinfo index cf77bec7b4e..a8450690b6f 100755 --- a/src/sonic-build-hooks/scripts/pre_run_buildinfo +++ b/src/sonic-build-hooks/scripts/pre_run_buildinfo @@ -9,7 +9,7 @@ mkdir -p $LOG_PATH [ -d $PRE_VERSION_PATH ] && rm -rf $PRE_VERSION_PATH collect_version_files $PRE_VERSION_PATH -set_build_hooks +symlink_build_hooks chmod -R a+rw $BUILDINFO_PATH diff --git a/src/sonic-build-hooks/scripts/set_build_hooks b/src/sonic-build-hooks/scripts/symlink_build_hooks similarity index 100% rename from src/sonic-build-hooks/scripts/set_build_hooks rename to src/sonic-build-hooks/scripts/symlink_build_hooks From c4d2a2f1784753817b0fcbb3df9eba92bcc36c73 Mon Sep 17 00:00:00 2001 From: xumia Date: Thu, 10 Dec 2020 10:24:21 +0000 Subject: [PATCH 15/18] Remove set_build_hooks --- build_debian.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/build_debian.sh b/build_debian.sh index e471ce967fb..0fa602ad293 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -573,7 +573,6 @@ sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT -sudo LANG=C chroot $FILESYSTEM_ROOT set_build_hooks -d ## Compress docker files pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd From 22429345cb49d4a1ceac905d12b459ec2e2eff1d Mon Sep 17 00:00:00 2001 From: xumia Date: Sat, 12 Dec 2020 07:22:33 +0000 Subject: [PATCH 16/18] Change docker trusted gpg files --- scripts/prepare_docker_buildinfo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index b5745285dba..3b881253cc1 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -25,7 +25,7 @@ fi DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo COPY ["buildinfo", "/usr/local/share/buildinfo"] RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb -RUN cp -rf /usr/local/share/buildinfo/trusted.gpg.d/* /etc/apt/trusted.gpg.d/ +COPY ["buildinfo/trusted.gpg.d/*", "/etc/apt/trusted.gpg.d"] RUN pre_run_buildinfo' # Add the auto-generate code if it is not added in the target Dockerfile From d188c6618de2279766f5e484e34303ca50a401e0 Mon Sep 17 00:00:00 2001 From: xumia Date: Sat, 12 Dec 2020 07:35:05 +0000 Subject: [PATCH 17/18] Fix docker build COPY directory name issue --- scripts/prepare_docker_buildinfo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 3b881253cc1..2e0ecf6f288 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -25,7 +25,7 @@ fi DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo COPY ["buildinfo", "/usr/local/share/buildinfo"] RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb -COPY ["buildinfo/trusted.gpg.d/*", "/etc/apt/trusted.gpg.d"] +COPY ["buildinfo/trusted.gpg.d/*", "/etc/apt/trusted.gpg.d/"] RUN pre_run_buildinfo' # Add the auto-generate code if it is not added in the target Dockerfile From c1b0107c2447e8d290f1b9e999c6588e24335d03 Mon Sep 17 00:00:00 2001 From: xumia Date: Mon, 14 Dec 2020 13:00:30 +0000 Subject: [PATCH 18/18] Move the trusted gpg files into the sonic-build-hooks package --- Makefile.work | 4 ++-- scripts/generate_buildinfo_config.sh | 10 ---------- scripts/prepare_docker_buildinfo.sh | 9 +-------- scripts/prepare_slave_container_buildinfo.sh | 16 +++++++++------- slave.mk | 2 +- src/sonic-build-hooks/Makefile | 4 +++- 6 files changed, 16 insertions(+), 29 deletions(-) diff --git a/Makefile.work b/Makefile.work index 54f5e4778de..8d28377ae19 100644 --- a/Makefile.work +++ b/Makefile.work @@ -246,7 +246,7 @@ endif endif @$(OVERLAY_MODULE_CHECK) - @pushd src/sonic-build-hooks; make all; popd + @pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd @cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_BASE_IMAGE)/buildinfo @docker inspect --type image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) &> /dev/null || \ { echo Image $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) not found. Building... ; \ @@ -266,7 +266,7 @@ else endif sonic-build-hooks: - @pushd src/sonic-build-hooks; make all; popd + @pushd src/sonic-build-hooks; TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) make all; popd @cp src/sonic-build-hooks/buildinfo/sonic-build-hooks* $(SLAVE_BASE_IMAGE)/buildinfo sonic-slave-base-build : sonic-build-hooks diff --git a/scripts/generate_buildinfo_config.sh b/scripts/generate_buildinfo_config.sh index 63429258cef..fe7657a6b6c 100755 --- a/scripts/generate_buildinfo_config.sh +++ b/scripts/generate_buildinfo_config.sh @@ -2,19 +2,9 @@ BUILDINFO_PATH=src/sonic-build-hooks -TRUSTED_GPG_PATH=$BUILDINFO_PATH/buildinfo/trusted.gpg.d BUILDINFO_CONFIG=$BUILDINFO_PATH/buildinfo/config/buildinfo.config -[ -d $TRUSTED_GPG_PATH ] && rm -rf $TRUSTED_GPG_PATH -mkdir -p $TRUSTED_GPG_PATH mkdir -p $BUILDINFO_PATH/buildinfo/config echo "PACKAGE_URL_PREFIX=$PACKAGE_URL_PREFIX" > $BUILDINFO_CONFIG echo "SONIC_VERSION_CONTROL_COMPONENTS=$SONIC_VERSION_CONTROL_COMPONENTS" >> $BUILDINFO_CONFIG - - -# Download trusted gpgs -for url in $(echo $TRUSTED_GPG_URLS | sed 's/[,;]/ /g') -do - wget -q "$url" -P "$TRUSTED_GPG_PATH/" -done diff --git a/scripts/prepare_docker_buildinfo.sh b/scripts/prepare_docker_buildinfo.sh index 2e0ecf6f288..aa3aaaa4bed 100755 --- a/scripts/prepare_docker_buildinfo.sh +++ b/scripts/prepare_docker_buildinfo.sh @@ -25,7 +25,6 @@ fi DOCKERFILE_PRE_SCRIPT='# Auto-Generated for buildinfo COPY ["buildinfo", "/usr/local/share/buildinfo"] RUN dpkg -i /usr/local/share/buildinfo/sonic-build-hooks_1.0_all.deb -COPY ["buildinfo/trusted.gpg.d/*", "/etc/apt/trusted.gpg.d/"] RUN pre_run_buildinfo' # Add the auto-generate code if it is not added in the target Dockerfile @@ -36,7 +35,7 @@ if [ ! -f $DOCKERFILE_TARGE ] || ! grep -q "Auto-Generated for buildinfo" $DOCKE awk -v text="${DOCKERFILE_PRE_SCRIPT}" -v linenumber=$LINE_NUMBER 'NR==linenumber{print text}1' $DOCKERFILE > $TEMP_FILE # Append the docker build script at the end of the docker file - echo "RUN post_run_buildinfo" >> $TEMP_FILE + echo -e "\nRUN post_run_buildinfo" >> $TEMP_FILE cat $TEMP_FILE > $DOCKERFILE_TARGE rm -f $TEMP_FILE @@ -45,12 +44,6 @@ fi # Copy the build info config cp -rf src/sonic-build-hooks/buildinfo/* $BUILDINFO_PATH -# Build the slave running config -if [ "$BUILD_SLAVE" == "y" ]; then - scripts/versions_manager.py generate -t "${BUILDINFO_PATH}/build/versions" -n "build-${IMAGENAME}" -d "$DISTRO" -a "$ARCH" - touch ${BUILDINFO_PATH}/build/versions/versions-deb -fi - # Generate the version lock files scripts/versions_manager.py generate -t "$BUILDINFO_VERSION_PATH" -n "$IMAGENAME" -d "$DISTRO" -a "$ARCH" diff --git a/scripts/prepare_slave_container_buildinfo.sh b/scripts/prepare_slave_container_buildinfo.sh index 8bc9c719a9e..d66bbe81610 100755 --- a/scripts/prepare_slave_container_buildinfo.sh +++ b/scripts/prepare_slave_container_buildinfo.sh @@ -1,19 +1,21 @@ #!/bin/bash SLAVE_DIR=$1 - -BUILDINFO=$SLAVE_DIR/buildinfo -BUILD_VERSIONS_PATH=$SLAVE_DIR/buildinfo/build/versions -VERSION_DEB_PREFERENCE=$BUILD_VERSIONS_PATH/01-versions-deb +ARCH=$2 +DISTRO=$3 # Enable the build hooks symlink_build_hooks +# Build the slave running config cp -rf $SLAVE_DIR/buildinfo/* /usr/local/share/buildinfo/ +. /usr/local/share/buildinfo/scripts/buildinfo_base.sh +# Build the slave version config [ -d /usr/local/share/buildinfo/versions ] && rm -rf /usr/local/share/buildinfo/versions -mkdir -p /usr/local/share/buildinfo/versions -cp -rf $BUILD_VERSIONS_PATH/* /usr/local/share/buildinfo/versions/ +scripts/versions_manager.py generate -t "/usr/local/share/buildinfo/versions" -n "build-${SLAVE_DIR}" -d "$DISTRO" -a "$ARCH" +touch ${BUILDINFO_PATH}/versions/versions-deb rm -f /etc/apt/preferences.d/01-versions-deb -[ -f $VERSION_DEB_PREFERENCE ] && cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ +([ "$ENABLE_VERSION_CONTROL_DEB" == "y" ] && [ -f $VERSION_DEB_PREFERENCE ]) && cp -f $VERSION_DEB_PREFERENCE /etc/apt/preferences.d/ +exit 0 diff --git a/slave.mk b/slave.mk index 621d098af61..b4a623f31ce 100644 --- a/slave.mk +++ b/slave.mk @@ -265,7 +265,7 @@ $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH)) endif # Overwrite the buildinfo in slave container -$(shell sudo scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR)) +$(shell sudo scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)) include Makefile.cache diff --git a/src/sonic-build-hooks/Makefile b/src/sonic-build-hooks/Makefile index f0ec282f8de..7221d7900e3 100644 --- a/src/sonic-build-hooks/Makefile +++ b/src/sonic-build-hooks/Makefile @@ -9,6 +9,7 @@ SYMBOL_LINKS = symlink_build_hooks post_run_buildinfo pre_run_buildinfo collect_ SONIC_BUILD_HOOKS_TARGET = $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS_PACKAGE) BUILD_ROOT_DIR = $(TMP_DIR)/$(SONIC_BUILD_HOOKS) DEBIAN_DIR = $(BUILD_ROOT_DIR)/DEBIAN +TRUSTED_GPG_PATH = $(BUILD_ROOT_DIR)/etc/apt/trusted.gpg.d INSTALL_PATH = $(BUILD_ROOT_DIR)/usr/local/share/buildinfo SYMBOL_LINK_PATH = $(BUILD_ROOT_DIR)/usr/sbin SCRIPTS_PATH = $(INSTALL_PATH)/scripts @@ -26,10 +27,11 @@ endif DEPENDS := $(shell find scripts hooks debian -type f) $(SONIC_BUILD_HOOKS_TARGET): $(DEPENDS) @rm -rf $(BUILDINFO_DIR)/$(SONIC_BUILD_HOOKS) $(TMP_DIR) - @mkdir -p $(DEBIAN_DIR) $(SCRIPTS_PATH) $(HOOKS_PATH) $(SYMBOL_LINK_PATH) + @mkdir -p $(DEBIAN_DIR) $(SCRIPTS_PATH) $(HOOKS_PATH) $(SYMBOL_LINK_PATH) $(TRUSTED_GPG_PATH) @cp debian/* $(DEBIAN_DIR)/ @cp scripts/* $(SCRIPTS_PATH)/ @cp hooks/* $(HOOKS_PATH)/ + @for url in $$(echo $(TRUSTED_GPG_URLS) | sed 's/[,;]/ /g'); do wget -q "$$url" -P "$(TRUSTED_GPG_PATH)/"; done @for f in $(SYMBOL_LINKS); do ln -s $(SYMBOL_LINKS_SRC_DIR)/$$f $(SYMBOL_LINK_PATH)/$$f; done @$(BUILD_COMMAND) @sudo chmod a+rw $(SONIC_BUILD_HOOKS_TARGET)