[Reland] Update dockerfiles to do staged builds

Makefile.work

@@ -293,8 +293,8 @@ endif
 DOCKER_LOCKFILE_SAVE := $(DOCKER_LOCKDIR)/docker_save.lock
 $(shell mkdir -m 0777 -p $(DOCKER_LOCKDIR))
 $(shell [ -f $(DOCKER_LOCKFILE_SAVE) ] || (touch $(DOCKER_LOCKFILE_SAVE) && chmod 0777 $(DOCKER_LOCKFILE_SAVE)))
-$(shell [ -d $(DOCKER_ROOT) ] && docker run --rm -v $(DOCKER_ROOT)\:/mount debian sh -c 'rm -rf /mount/*')
-$(mkdir -p $(DOCKER_ROOT))
+$(shell [ -d $(DOCKER_ROOT) ] && docker run --rm -v $(DOCKER_ROOT)\:/mount $(DEFAULT_CONTAINER_REGISTRY)debian:bookworm sh -c 'rm -rf /mount/*')
+$(shell mkdir -p $(DOCKER_ROOT))
 
 ifeq ($(DOCKER_BUILDER_MOUNT),)
 override DOCKER_BUILDER_MOUNT := "$(PWD):/sonic"
@@ -304,6 +304,8 @@ ifeq ($(DOCKER_BUILDER_WORKDIR),)
 override DOCKER_BUILDER_WORKDIR := "/sonic"
 endif
 
+# Consider removing the --ulimit flag once nothing older
+# than Bullseye is being used as a slave container.
 DOCKER_RUN := docker run --rm=true --privileged --init \
     -v $(DOCKER_BUILDER_MOUNT) \
     -v "$(DOCKER_LOCKDIR):$(DOCKER_LOCKDIR)" \
@@ -312,6 +314,7 @@ DOCKER_RUN := docker run --rm=true --privileged --init \
     -e "https_proxy=$(https_proxy)" \
     -e "no_proxy=$(no_proxy)" \
     -i$(shell { if [ -t 0 ]; then echo t; fi }) \
+    --ulimit nofile=524288:524288 \
     $(SONIC_BUILDER_EXTRA_CMDLINE)
 
 # Mount the $(DOCKER_ROOT) to /var/lib/docker in the slave container, the overlay fs is not supported as dockerd root folder.

build_debug_docker_j2.sh

@@ -1,7 +1,8 @@
 #! /bin/bash
 
 echo "
-FROM $1
+ARG BASE=$1
+FROM \$BASE AS base
 
 ARG docker_container_name
 
@@ -41,4 +42,8 @@ RUN apt-get update && apt-get install -f -y \
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /debs
 
+FROM \$BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=/proc --exclude=/dev --exclude=resolv.conf /changes-to-image/ /
+
 "

dockers/docker-auditd-watchdog/Dockerfile.j2

@@ -1,4 +1,7 @@
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}} AS builder
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+ARG BASE=docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE AS builder
 
 # Update apt's cache of available packages
 RUN apt-get update && apt-get install -y \
@@ -18,7 +21,7 @@ COPY watchdog/ ./
 # Build from within /watchdog
 RUN cargo build --release
 
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+FROM $BASE AS base
 
 ARG docker_container_name
 ARG image_version
@@ -27,13 +30,21 @@ RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%s
 ENV DEBIAN_FRONTEND=noninteractive
 ENV IMAGE_VERSION=$image_version
 
-RUN apt-get update
-
 # Copy supervisord.conf into final stage
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 
 # Copy the compiled Rust binary from the builder stage
 COPY --from=builder /watchdog/target/release/watchdog /usr/bin/auditd_watchdog
 RUN chmod +x /usr/bin/auditd_watchdog
 
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=/proc --exclude=/dev --exclude=resolv.conf /changes-to-image/ /
+
+# Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
 ENTRYPOINT ["/usr/local/bin/supervisord"]

dockers/docker-auditd/Dockerfile.j2

@@ -1,4 +1,7 @@
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+ARG BASE=docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE AS base
 
 ARG docker_container_name
 ARG image_version
@@ -10,13 +13,20 @@ ENV DEBIAN_FRONTEND=noninteractive
 # Pass the image_version to container
 ENV IMAGE_VERSION=$image_version
 
-# Update apt's cache of available packages
-RUN apt-get update
-
 COPY ["config_checker.py", "/usr/bin/"]
 COPY ["auditd_config_files/", "/usr/share/sonic/auditd_config_files/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 
 RUN chmod +x /usr/bin/config_checker.py
 
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=/proc --exclude=/dev --exclude=resolv.conf /changes-to-image/ /
+
+# Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
 ENTRYPOINT ["/usr/local/bin/supervisord"]

dockers/docker-base-bookworm/Dockerfile.j2

@@ -1,26 +1,14 @@
 {% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
 {% if CONFIGURED_ARCH == "armhf" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
-FROM --platform=linux/arm/v7 {{ prefix }}debian:bookworm
+ARG BASE=--platform=linux/arm/v7 {{ prefix }}debian:bookworm
 {% elif CONFIGURED_ARCH == "arm64" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
-FROM --platform=linux/arm64 {{ prefix }}debian:bookworm
+ARG BASE=--platform=linux/arm64 {{ prefix }}debian:bookworm
 {% else %}
-FROM {{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bookworm
+ARG BASE={{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bookworm
 {% endif %}
 
-# Clean documentation in FROM image
-RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true
-
-# Clean doc directories that are empty or only contain empty directories
-RUN while [ -n "$(find /usr/share/doc -depth -type d -empty -print -exec rmdir {} +)" ]; do :; done && \
-    rm -rf               \
-    /usr/share/man/*     \
-    /usr/share/groff/*   \
-    /usr/share/info/*    \
-    /usr/share/lintian/* \
-    /usr/share/linda/*   \
-    /var/cache/man/*     \
-    /usr/share/locale/*
+FROM $BASE AS base
 
 # Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -47,6 +35,8 @@ RUN apt update &&            \
         python-is-python3    \
         vim-tiny             \
         rsyslog              \
+# Install rsync for copying over only changes between layers
+        rsync                \
 # Install redis-tools
         redis-tools          \
 # common dependencies
@@ -84,12 +74,6 @@ RUN mkdir -p /var/log/supervisor /etc/supervisor/conf.d
 # Uninstall unused dependencies
 RUN apt autoremove -y --purge
 
-RUN apt-get -y purge   \
-    exim4              \
-    exim4-base         \
-    exim4-config       \
-    exim4-daemon-light
-
 {% if docker_base_bookworm_debs.strip() -%}
 # Copy locally-built Debian package dependencies
 {{ copy_files("debs/", docker_base_bookworm_debs.split(' '), "/debs/") }}
@@ -98,17 +82,14 @@ RUN apt-get -y purge   \
 {{ install_debian_packages(docker_base_bookworm_debs.split(' ')) }}
 {%- endif %}
 
-# Clean up apt
-# Remove /var/lib/apt/lists/*, could be obsoleted for derived images
-RUN apt-get clean -y                     && \
-    apt-get autoclean -y                 && \
-    apt-get autoremove -y                && \
-    rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache
-
 COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
 RUN ln /usr/bin/vim.tiny /usr/bin/vim
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+
+FROM scratch
+
+COPY --from=base / /

dockers/docker-base-bullseye/Dockerfile.j2

@@ -1,26 +1,14 @@
 {% set prefix = DEFAULT_CONTAINER_REGISTRY %}
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
 {% if CONFIGURED_ARCH == "armhf" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
-FROM {{ prefix }}multiarch/debian-debootstrap:armhf-bullseye
+ARG BASE={{ prefix }}multiarch/debian-debootstrap:armhf-bullseye
 {% elif CONFIGURED_ARCH == "arm64" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
-FROM {{ prefix }}multiarch/debian-debootstrap:arm64-bullseye
+ARG BASE={{ prefix }}multiarch/debian-debootstrap:arm64-bullseye
 {% else %}
-FROM {{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bullseye
+ARG BASE={{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bullseye
 {% endif %}
 
-# Clean documentation in FROM image
-RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true
-
-# Clean doc directories that are empty or only contain empty directories
-RUN while [ -n "$(find /usr/share/doc -depth -type d -empty -print -exec rmdir {} +)" ]; do :; done && \
-    rm -rf               \
-    /usr/share/man/*     \
-    /usr/share/groff/*   \
-    /usr/share/info/*    \
-    /usr/share/lintian/* \
-    /usr/share/linda/*   \
-    /var/cache/man/*     \
-    /usr/share/locale/*
+FROM $BASE AS base
 
 # Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -45,6 +33,8 @@ RUN apt-get update &&        \
         python3-pip          \
         python-is-python3    \
         vim-tiny             \
+# Install rsync for copying over only changes between layers
+        rsync                \
 # Install redis-tools
         redis-tools          \
 # common dependencies
@@ -100,17 +90,14 @@ RUN apt-get -y purge   \
 {{ install_debian_packages(docker_base_bullseye_debs.split(' ')) }}
 {%- endif %}
 
-# Clean up apt
-# Remove /var/lib/apt/lists/*, could be obsoleted for derived images
-RUN apt-get clean -y                     && \
-    apt-get autoclean -y                 && \
-    apt-get autoremove -y                && \
-    rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache
-
 COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
 COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
 COPY ["root/.vimrc", "/root/.vimrc"]
 
 RUN ln /usr/bin/vim.tiny /usr/bin/vim
 
 COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
+
+FROM scratch
+
+COPY --from=base / /

dockers/docker-bmp-watchdog/Dockerfile.j2

@@ -1,4 +1,7 @@
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}} AS builder
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+ARG BASE=docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE AS builder
 
 # Update apt's cache of available packages
 RUN apt-get update && apt-get install -y \
@@ -18,7 +21,7 @@ COPY watchdog/ ./
 # Build from within /watchdog
 RUN cargo build --release
 
-FROM docker-config-engine-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+FROM $BASE AS base
 
 ARG docker_container_name
 ARG image_version
@@ -27,13 +30,21 @@ RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%s
 ENV DEBIAN_FRONTEND=noninteractive
 ENV IMAGE_VERSION=$image_version
 
-RUN apt-get update
-
 # Copy supervisord.conf into final stage
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 
 # Copy the compiled Rust binary from the builder stage
 COPY --from=builder /watchdog/target/release/bmp_watchdog /usr/bin/bmp_watchdog
 RUN chmod +x /usr/bin/bmp_watchdog
 
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=/proc --exclude=/dev --exclude=resolv.conf /changes-to-image/ /
+
+# Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Pass the image_version to container
+ENV IMAGE_VERSION=$image_version
+
 ENTRYPOINT ["/usr/local/bin/supervisord"]

dockers/docker-config-engine-bookworm/Dockerfile.j2

@@ -1,5 +1,7 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-base-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE AS base
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -49,10 +51,17 @@ COPY ["files/readiness_probe.sh", "/usr/bin/"]
 COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
 
 ## Clean up
-RUN apt-get purge -y               \
-        python3-dev                \
-        build-essential         && \
-    apt-get clean -y            && \
-    apt-get autoclean -y        && \
-    apt-get autoremove -y       && \
-    rm -rf /debs /python-wheels ~/.cache
+
+{%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %}
+RUN apt-get purge -y        \
+        libxslt-dev         \
+        libz-dev
+{%- endif %}
+
+RUN apt-get purge -y        \
+        python3-dev         \
+        build-essential
+
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=/proc --exclude=/dev --exclude=resolv.conf /changes-to-image/ /

dockers/docker-config-engine-bullseye/Dockerfile.j2

@@ -1,5 +1,7 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-base-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-base-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE AS base
 
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -55,3 +57,7 @@ RUN apt-get purge -y               \
     apt-get autoclean -y        && \
     apt-get autoremove -y       && \
     rm -rf /debs /python-wheels ~/.cache
+
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=/proc --exclude=/dev --exclude=resolv.conf /changes-to-image/ /

dockers/docker-dash-ha/Dockerfile.j2

@@ -1,5 +1,7 @@
 {% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
-FROM docker-swss-layer-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+ARG BASE=docker-swss-layer-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+FROM $BASE AS base
 
 ARG docker_container_name
 
@@ -15,13 +17,14 @@ RUN apt-get update
 {{ install_debian_packages(docker_dash_ha_debs.split(' ')) }}
 {%- endif %}
 
-RUN apt-get clean -y      && \
-    apt-get autoclean -y  && \
-    apt-get autoremove -y && \
-    rm -rf /debs
-
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["critical_processes", "/etc/supervisor"]
 
+FROM $BASE
+
+RUN --mount=type=bind,from=base,target=/changes-to-image rsync -axAX --no-D --exclude=/sys --exclude=/proc --exclude=/dev --exclude=resolv.conf /changes-to-image/ /
+
+ENV DEBIAN_FRONTEND=noninteractive
+
 ENTRYPOINT ["/usr/local/bin/supervisord"]