From d6cc8e691d1e86b108836e19a05e6cffab726cee Mon Sep 17 00:00:00 2001 From: xumia Date: Thu, 26 May 2022 06:13:31 +0000 Subject: [PATCH 1/5] Revert "Revert "Reduce image size for lazy installation packages (#10775)" (#10916)" This reverts commit f0dfd398a6ed425034d8a80231d7bfd4880a4a50. --- build_debian.sh | 2 +- files/build_templates/sonic_debian_extension.j2 | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build_debian.sh b/build_debian.sh index 25f039c9069..676c0298ec7 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -647,5 +647,5 @@ fi pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd ## Compress together with /boot, /var/lib/docker and $PLATFORM_DIR as an installer payload zip file -pushd $FILESYSTEM_ROOT && sudo zip $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ $PLATFORM_DIR/; popd +pushd $FILESYSTEM_ROOT && sudo zip --symlinks $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ $PLATFORM_DIR/; popd sudo zip -g -n .squashfs:.gz $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index fc66c74f865..760ef2ebc42 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -614,7 +614,9 @@ sudo LANG=C chroot $FILESYSTEM_ROOT depmod -a {{kversion}} sudo dpkg --root=$FILESYSTEM_ROOT -i {{deb}} || sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f --download-only sudo mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}} -sudo cp {{ deb }} $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}/ +sudo mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/common +sudo cp {{ deb }} $FILESYSTEM_ROOT/$PLATFORM_DIR/common/ +sudo ln -sf "../common/{{ debfilename }}" "$FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}/{{ debfilename }}" for f in $(find $FILESYSTEM_ROOT/var/cache/apt/archives -name "*.deb"); do sudo mv $f $FILESYSTEM_ROOT/$PLATFORM_DIR/{{dev}}/ done From 96c8fba1342b5f657762b47cd8bcbb5ec22b0dd3 Mon Sep 17 00:00:00 2001 From: xumia Date: Thu, 26 May 2022 06:15:31 +0000 Subject: [PATCH 2/5] Use the platform.tar.gz as a replacement for platform folder --- build_debian.sh | 2 +- files/Aboot/boot0.j2 | 6 ++++++ installer/arm64/install.sh | 3 +++ installer/armhf/install.sh | 3 +++ installer/x86_64/install.sh | 4 ++++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/build_debian.sh b/build_debian.sh index 676c0298ec7..5ad8e38a5d3 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -647,5 +647,5 @@ fi pushd $FILESYSTEM_ROOT && sudo tar czf $OLDPWD/$FILESYSTEM_DOCKERFS -C ${DOCKERFS_PATH}var/lib/docker .; popd ## Compress together with /boot, /var/lib/docker and $PLATFORM_DIR as an installer payload zip file -pushd $FILESYSTEM_ROOT && sudo zip --symlinks $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ $PLATFORM_DIR/; popd +pushd $FILESYSTEM_ROOT && sudo tar czf platform.tar.gz -C $PLATFORM_DIR . && sudo zip -n .gz $OLDPWD/$ONIE_INSTALLER_PAYLOAD -r boot/ platform.tar.gz; popd sudo zip -g -n .squashfs:.gz $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS $FILESYSTEM_DOCKERFS diff --git a/files/Aboot/boot0.j2 b/files/Aboot/boot0.j2 index ad7540df094..0b372b16142 100644 --- a/files/Aboot/boot0.j2 +++ b/files/Aboot/boot0.j2 @@ -350,6 +350,12 @@ extract_image() { ## Unzip the image except boot0 and dockerfs archive unzip -oq "$swipath" -x boot0 "$dockerfs" -d "$image_path" + ## Extrat the platform.tar.gz + info "Extracting platform.tar.gz" + mkdir -p $image_path/platform + tar xzf $image_path/platform.tar.gz -C $image_path/platform + rm -rf $image_path/platform.tar.gz + ## detect rootfs type local mountstr="$(grep " $target_path " /proc/mounts)" local rootdev="$(echo $mountstr | cut -f1 -d' ')" diff --git a/installer/arm64/install.sh b/installer/arm64/install.sh index 445b2007fae..f9e42382e6e 100755 --- a/installer/arm64/install.sh +++ b/installer/arm64/install.sh @@ -168,6 +168,9 @@ else unzip -op $ONIE_INSTALLER_PAYLOAD "$FILESYSTEM_DOCKERFS" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/$DOCKERFS_DIR fi +mkdir -p $demo_mnt/$image_dir/platform +tar xzf $demo_mnt/$image_dir/platform.tar.gz -C $demo_mnt/$image_dir/platform +rm -rf $demo_mnt/$image_dir/platform.tar.gz if [ "$install_env" = "onie" ]; then # Store machine description in target file system diff --git a/installer/armhf/install.sh b/installer/armhf/install.sh index 0dd6e48a08e..8fb32a3c3f6 100755 --- a/installer/armhf/install.sh +++ b/installer/armhf/install.sh @@ -168,6 +168,9 @@ else unzip -op $ONIE_INSTALLER_PAYLOAD "$FILESYSTEM_DOCKERFS" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/$DOCKERFS_DIR fi +mkdir -p $demo_mnt/$image_dir/platform +tar xzf $demo_mnt/$image_dir/platform.tar.gz -C $demo_mnt/$image_dir/platform +rm -rf $demo_mnt/$image_dir/platform.tar.gz if [ "$install_env" = "onie" ]; then # Store machine description in target file system diff --git a/installer/x86_64/install.sh b/installer/x86_64/install.sh index f41a671543e..0765df54a74 100755 --- a/installer/x86_64/install.sh +++ b/installer/x86_64/install.sh @@ -549,6 +549,10 @@ else unzip -op $ONIE_INSTALLER_PAYLOAD "$FILESYSTEM_DOCKERFS" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/$DOCKERFS_DIR fi +mkdir -p $demo_mnt/$image_dir/platform +tar xzf $demo_mnt/$image_dir/platform.tar.gz -C $demo_mnt/$image_dir/platform +rm -rf $demo_mnt/$image_dir/platform.tar.gz + if [ "$install_env" = "onie" ]; then # Store machine description in target file system if [ -f /etc/machine-build.conf ]; then From 677c5bfd3295c0be0cee804d45e353d094a9b095 Mon Sep 17 00:00:00 2001 From: xumia Date: Wed, 8 Jun 2022 10:52:36 +0000 Subject: [PATCH 3/5] Reduce the requird disk size to unzip platform.tar.gz --- files/Aboot/boot0.j2 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/files/Aboot/boot0.j2 b/files/Aboot/boot0.j2 index 0b372b16142..2efdb4cf1c3 100644 --- a/files/Aboot/boot0.j2 +++ b/files/Aboot/boot0.j2 @@ -348,13 +348,12 @@ extract_image() { info "Extracting swi content" ## Unzip the image except boot0 and dockerfs archive - unzip -oq "$swipath" -x boot0 "$dockerfs" -d "$image_path" + unzip -oq "$swipath" -x boot0 "$dockerfs" "platform.tar.gz" -d "$image_path" ## Extrat the platform.tar.gz info "Extracting platform.tar.gz" - mkdir -p $image_path/platform - tar xzf $image_path/platform.tar.gz -C $image_path/platform - rm -rf $image_path/platform.tar.gz + mkdir -p "$image_path/platform" + unzip -oqp "$swipath" "platform.tar.gz" | tar xzf - -C "$image_path/platform" $TAR_EXTRA_OPTION ## detect rootfs type local mountstr="$(grep " $target_path " /proc/mounts)" From b6039a7342a39cdb74fd1aa1e2ac5c6fef73e025 Mon Sep 17 00:00:00 2001 From: xumia Date: Wed, 8 Jun 2022 10:57:39 +0000 Subject: [PATCH 4/5] Fix a type issue --- files/Aboot/boot0.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/Aboot/boot0.j2 b/files/Aboot/boot0.j2 index 2efdb4cf1c3..e6dac1101fb 100644 --- a/files/Aboot/boot0.j2 +++ b/files/Aboot/boot0.j2 @@ -350,7 +350,7 @@ extract_image() { ## Unzip the image except boot0 and dockerfs archive unzip -oq "$swipath" -x boot0 "$dockerfs" "platform.tar.gz" -d "$image_path" - ## Extrat the platform.tar.gz + ## Extract the platform.tar.gz info "Extracting platform.tar.gz" mkdir -p "$image_path/platform" unzip -oqp "$swipath" "platform.tar.gz" | tar xzf - -C "$image_path/platform" $TAR_EXTRA_OPTION From dff8acb7f332b8d25a67b7618d1e6f9aa4ad527f Mon Sep 17 00:00:00 2001 From: xumia Date: Wed, 8 Jun 2022 14:07:05 +0000 Subject: [PATCH 5/5] Change to extract tar files in memory --- installer/arm64/install.sh | 7 +++---- installer/armhf/install.sh | 7 +++---- installer/x86_64/install.sh | 7 +++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/installer/arm64/install.sh b/installer/arm64/install.sh index f9e42382e6e..54ce1dda934 100755 --- a/installer/arm64/install.sh +++ b/installer/arm64/install.sh @@ -155,9 +155,9 @@ fi # Decompress the file for the file system directly to the partition if [ x"$docker_inram" = x"on" ]; then # when disk is small, keep dockerfs.tar.gz in disk, expand it into ramfs during initrd - unzip -o $ONIE_INSTALLER_PAYLOAD -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "platform.tar.gz" -d $demo_mnt/$image_dir else - unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" "platform.tar.gz" -d $demo_mnt/$image_dir if [ "$install_env" = "onie" ]; then TAR_EXTRA_OPTION="--numeric-owner" @@ -169,8 +169,7 @@ else fi mkdir -p $demo_mnt/$image_dir/platform -tar xzf $demo_mnt/$image_dir/platform.tar.gz -C $demo_mnt/$image_dir/platform -rm -rf $demo_mnt/$image_dir/platform.tar.gz +unzip -op $ONIE_INSTALLER_PAYLOAD "platform.tar.gz" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/platform if [ "$install_env" = "onie" ]; then # Store machine description in target file system diff --git a/installer/armhf/install.sh b/installer/armhf/install.sh index 8fb32a3c3f6..4ced27f48fa 100755 --- a/installer/armhf/install.sh +++ b/installer/armhf/install.sh @@ -155,9 +155,9 @@ fi # Decompress the file for the file system directly to the partition if [ x"$docker_inram" = x"on" ]; then # when disk is small, keep dockerfs.tar.gz in disk, expand it into ramfs during initrd - unzip -o $ONIE_INSTALLER_PAYLOAD -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "platform.tar.gz" -d $demo_mnt/$image_dir else - unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" "platform.tar.gz" -d $demo_mnt/$image_dir if [ "$install_env" = "onie" ]; then TAR_EXTRA_OPTION="--numeric-owner" @@ -169,8 +169,7 @@ else fi mkdir -p $demo_mnt/$image_dir/platform -tar xzf $demo_mnt/$image_dir/platform.tar.gz -C $demo_mnt/$image_dir/platform -rm -rf $demo_mnt/$image_dir/platform.tar.gz +unzip -op $ONIE_INSTALLER_PAYLOAD "platform.tar.gz" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/platform if [ "$install_env" = "onie" ]; then # Store machine description in target file system diff --git a/installer/x86_64/install.sh b/installer/x86_64/install.sh index 0765df54a74..926c54683a9 100755 --- a/installer/x86_64/install.sh +++ b/installer/x86_64/install.sh @@ -536,9 +536,9 @@ fi # Decompress the file for the file system directly to the partition if [ x"$docker_inram" = x"on" ]; then # when disk is small, keep dockerfs.tar.gz in disk, expand it into ramfs during initrd - unzip -o $ONIE_INSTALLER_PAYLOAD -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "platform.tar.gz" -d $demo_mnt/$image_dir else - unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" -d $demo_mnt/$image_dir + unzip -o $ONIE_INSTALLER_PAYLOAD -x "$FILESYSTEM_DOCKERFS" "platform.tar.gz" -d $demo_mnt/$image_dir if [ "$install_env" = "onie" ]; then TAR_EXTRA_OPTION="--numeric-owner" @@ -550,8 +550,7 @@ else fi mkdir -p $demo_mnt/$image_dir/platform -tar xzf $demo_mnt/$image_dir/platform.tar.gz -C $demo_mnt/$image_dir/platform -rm -rf $demo_mnt/$image_dir/platform.tar.gz +unzip -op $ONIE_INSTALLER_PAYLOAD "platform.tar.gz" | tar xz $TAR_EXTRA_OPTION -f - -C $demo_mnt/$image_dir/platform if [ "$install_env" = "onie" ]; then # Store machine description in target file system