[docker-ptf]: consolidate docker-ptf build

.artifactignore

@@ -3,3 +3,4 @@
 !target/*.log
 !target/*.img.gz
 !target/docker-sonic-vs.gz
+!target/docker-ptf.gz

azure-pipelines.yml

@@ -78,7 +78,7 @@ stages:
         ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=vs
         trap "sudo rm -rf fsroot" EXIT
         make USERNAME=admin SONIC_BUILD_JOBS=$(nproc) $CACHE_OPTIONS \
-            target/docker-sonic-vs.gz target/sonic-vs.img.gz && \
+            target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz && \
             sudo cp target/sonic-vs.img.gz /nfs/azpl/kvmimage/sonic-vs.$(Build.BuildNumber).img.gz && \
             sudo cp target/docker-sonic-vs.gz /nfs/azpl/kvmimage/docker-sonic-vs.$(Build.BuildNumber).gz
       displayName: 'Build sonic image'

dockers/docker-ptf/Dockerfile.j2

@@ -78,7 +78,7 @@ RUN rm -rf /debs \
     && apt-get -y autoclean \
     && apt-get -y autoremove \
     && rm -rf /var/lib/apt/lists/* \
-    && wget --https-only https://bootstrap.pypa.io/get-pip.py \
+    && wget --https-only https://bootstrap.pypa.io/2.7/get-pip.py \
     && python get-pip.py \
     && rm -f get-pip.py \
     && pip install setuptools \
@@ -146,6 +146,18 @@ RUN git clone https://github.com/lguohan/gnxi.git \
     && cd gnmi_cli_py \
     && pip install -r requirements.txt
 
+COPY \
+{% for deb in docker_ptf_debs.split(' ') -%}
+debs/{{ deb }}{{' '}}
+{%- endfor -%}
+debs/
+
+RUN dpkg -i \
+{% for deb in docker_ptf_debs.split(' ') -%}
+debs/{{ deb }}{{' '}}
+{%- endfor %}
+
+COPY ["*.ini", "/etc/ptf/"]
 EXPOSE 22 8009
 
 ENTRYPOINT ["/usr/local/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]

platform/barefoot/rules.mk

@@ -10,7 +10,6 @@ include $(PLATFORM_PATH)/docker-syncd-bfn-rpc.mk
 include $(PLATFORM_PATH)/one-aboot.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-bfn.mk
 include $(PLATFORM_PATH)/bfn-platform.mk
 #include $(PLATFORM_PATH)/bfn-platform-wnc.mk
 #include $(PLATFORM_PATH)/bfn-platform-ingrasys.mk

platform/broadcom/rules.dep

@@ -22,4 +22,3 @@ include $(PLATFORM_PATH)/one-pde-image.dep
 include $(PLATFORM_PATH)/raw-image.dep
 include $(PLATFORM_PATH)/one-aboot.dep
 include $(PLATFORM_PATH)/libsaithrift-dev.dep
-include $(PLATFORM_PATH)/docker-ptf-brcm.dep

platform/broadcom/rules.mk

@@ -19,7 +19,6 @@ include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/raw-image.mk
 include $(PLATFORM_PATH)/one-aboot.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-brcm.mk
 
 BCMCMD = bcmcmd
 $(BCMCMD)_URL = "https://sonicstorage.blob.core.windows.net/packages/20190307/bcmcmd?sv=2015-04-05&sr=b&sig=sUdbU7oVbh5exbXXHVL5TDFBTWDDBASHeJ8Cp0B0TIc%3D&se=2038-05-06T22%3A34%3A19Z&sp=r"

platform/cavium/rules.mk

@@ -5,7 +5,6 @@ include $(PLATFORM_PATH)/cavm-platform-modules.mk
 include $(PLATFORM_PATH)/cavm-xpnet.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-cavm.mk
 
 SONIC_ALL += $(SONIC_ONE_IMAGE) \
              $(DOCKER_FPM) \

platform/centec-arm64/rules.mk

@@ -3,7 +3,6 @@ include $(PLATFORM_PATH)/docker-syncd-centec.mk
 include $(PLATFORM_PATH)/docker-syncd-centec-rpc.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-centec.mk
 include $(PLATFORM_PATH)/tsingma-bsp.mk
 include $(PLATFORM_PATH)/platform-modules-centec-e530.mk
 

platform/centec/rules.mk

@@ -5,7 +5,6 @@ include $(PLATFORM_PATH)/docker-syncd-centec.mk
 include $(PLATFORM_PATH)/docker-syncd-centec-rpc.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-centec.mk
 
 SONIC_ALL += $(SONIC_ONE_IMAGE)
 

platform/innovium/rules.mk

@@ -6,7 +6,6 @@ include $(PLATFORM_PATH)/docker-syncd-invm-rpc.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
 include $(PLATFORM_PATH)/python-saithrift.mk
-include $(PLATFORM_PATH)/docker-ptf-invm.mk
 
 SONIC_ALL += $(SONIC_INVM_ONE_IMAGE) \
              $(DOCKER_FPM) \

platform/marvell-arm64/rules.mk

@@ -4,7 +4,6 @@ include $(PLATFORM_PATH)/docker-syncd-mrvl.mk
 include $(PLATFORM_PATH)/docker-syncd-mrvl-rpc.mk
 include $(PLATFORM_PATH)/docker-saiserver-mrvl.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-mrvl.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/linux-kernel-arm64.mk
 INCLUDE_SYSTEM_TELEMETRY = ""

platform/marvell-armhf/rules.mk

@@ -4,7 +4,6 @@ include $(PLATFORM_PATH)/docker-syncd-mrvl.mk
 include $(PLATFORM_PATH)/docker-syncd-mrvl-rpc.mk
 include $(PLATFORM_PATH)/docker-saiserver-mrvl.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-mrvl.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/linux-kernel-armhf.mk
 include $(PLATFORM_PATH)/platform-et6448m.mk

platform/mellanox/rules.dep

@@ -11,7 +11,6 @@ include $(PLATFORM_PATH)/docker-syncd-mlnx-rpc.dep
 include $(PLATFORM_PATH)/docker-saiserver-mlnx.dep
 include $(PLATFORM_PATH)/one-image.dep
 include $(PLATFORM_PATH)/libsaithrift-dev.dep
-include $(PLATFORM_PATH)/docker-ptf-mlnx.dep
 include $(PLATFORM_PATH)/mlnx-ffb.dep
 include $(PLATFORM_PATH)/issu-version.dep
 include $(PLATFORM_PATH)/mlnx-onie-fw-update.dep

platform/mellanox/rules.mk

@@ -9,7 +9,6 @@ include $(PLATFORM_PATH)/docker-syncd-mlnx-rpc.mk
 include $(PLATFORM_PATH)/docker-saiserver-mlnx.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-mlnx.mk
 include $(PLATFORM_PATH)/mlnx-ffb.mk
 include $(PLATFORM_PATH)/issu-version.mk
 include $(PLATFORM_PATH)/mlnx-onie-fw-update.mk

platform/nephos/rules.mk

@@ -7,7 +7,6 @@ include $(PLATFORM_PATH)/docker-syncd-nephos.mk
 include $(PLATFORM_PATH)/docker-syncd-nephos-rpc.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/libsaithrift-dev.mk
-include $(PLATFORM_PATH)/docker-ptf-nephos.mk
 
 NPX_DIAG = npx_diag
 $(NPX_DIAG)_URL = "https://github.com/NephosInc/SONiC/raw/master/sdk/npx_diag"

rules/docker-ptf.dep → platform/vs/docker-ptf.dep

@@ -1,10 +1,10 @@
 
 DPATH       := $($(DOCKER_PTF)_PATH)
-DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-ptf.mk rules/docker-ptf.dep   
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) platform/vs/docker-ptf.mk platform/vs/docker-ptf.dep
 DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
 DEP_FILES   += $(shell git ls-files $(DPATH))
 
-$(DOCKER_PTF)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_PTF)_CACHE_MODE  := GIT_CONTENT_SHA
 $(DOCKER_PTF)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
 $(DOCKER_PTF)_DEP_FILES   := $(DEP_FILES)
 

rules/docker-ptf.mk → platform/vs/docker-ptf.mk

@@ -1,7 +1,7 @@
-# docker image for ptf
+# docker image for docker-ptf
 
 DOCKER_PTF = docker-ptf.gz
 $(DOCKER_PTF)_PATH = $(DOCKERS_PATH)/docker-ptf
-$(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF)
+$(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF) $(PYTHON_SAITHRIFT)
 SONIC_DOCKER_IMAGES += $(DOCKER_PTF)
 SONIC_STRETCH_DOCKERS += $(DOCKER_PTF)

platform/vs/libsaithrift-dev.dep

@@ -0,0 +1,13 @@
+#DPKG FRK
+SPATH       := $($(LIBSAITHRIFT_DEV)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) platform/broadcom/libsaithrift-dev.mk platform/broadcom/libsaithrift-dev.dep
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_PATHS := $(SPATH)  $(SPATH)/bm/behavioral-model  $(SPATH)/test/ptf  $(SPATH)/test/saithrift/ctypesgen
+$(foreach path, $(SMDEP_PATHS), $(eval $(path) :=$(filter-out $(SMDEP_PATHS),$(addprefix $(path)/, $(shell cd $(path) &&  git ls-files | grep -Ev " " )))))
+
+$(LIBSAITHRIFT_DEV)_CACHE_MODE  := GIT_CONTENT_SHA
+$(LIBSAITHRIFT_DEV)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(LIBSAITHRIFT_DEV)_DEP_FILES   := $(DEP_FILES)
+$(LIBSAITHRIFT_DEV)_SMDEP_FILES := $(foreach path, $(SMDEP_PATHS), $($(path)))
+$(LIBSAITHRIFT_DEV)_SMDEP_PATHS := $(SMDEP_PATHS)
+

platform/vs/libsaithrift-dev.mk

@@ -0,0 +1,22 @@
+# libsaithrift-dev package
+
+SAI_VER = 0.9.4
+
+LIBSAITHRIFT_DEV = libsaithrift-dev_$(SAI_VER)_$(CONFIGURED_ARCH).deb
+$(LIBSAITHRIFT_DEV)_SRC_PATH = $(SRC_PATH)/sonic-sairedis/SAI
+$(LIBSAITHRIFT_DEV)_DEPENDS += $(LIBTHRIFT) $(LIBTHRIFT_DEV) $(PYTHON_THRIFT) $(THRIFT_COMPILER) \
+	$(LIBSAIVS) $(LIBSAIVS_DEV) $(LIBSAIMETADATA) $(LIBSAIMETADATA_DEV)
+$(LIBSAITHRIFT_DEV)_RDEPENDS += $(LIBTHRIFT) $(LIBSAIVS) $(LIBSAIMETADATA)
+$(LIBSAITHRIFT_DEV)_BUILD_ENV = platform=vs
+SONIC_DPKG_DEBS += $(LIBSAITHRIFT_DEV)
+
+PYTHON_SAITHRIFT = python-saithrift_$(SAI_VER)_$(CONFIGURED_ARCH).deb
+$(eval $(call add_extra_package,$(LIBSAITHRIFT_DEV),$(PYTHON_SAITHRIFT)))
+
+SAISERVER = saiserver_$(SAI_VER)_$(CONFIGURED_ARCH).deb
+$(SAISERVER)_RDEPENDS += $(LIBTHRIFT) $(LIBSAIVS)
+$(eval $(call add_extra_package,$(LIBSAITHRIFT_DEV),$(SAISERVER)))
+
+SAISERVER_DBG = saiserver-dbg_$(SAI_VER)_$(CONFIGURED_ARCH).deb
+$(SAISERVER_DBG)_RDEPENDS += $(SAISERVER)
+$(eval $(call add_extra_package,$(LIBSAITHRIFT_DEV),$(SAISERVER_DBG)))

platform/vs/rules.dep

@@ -3,6 +3,8 @@ include $(PLATFORM_PATH)/sonic-version.dep
 include $(PLATFORM_PATH)/docker-sonic-vs.dep
 include $(PLATFORM_PATH)/docker-syncd-vs.dep
 include $(PLATFORM_PATH)/docker-gbsyncd-vs.dep
+include $(PLATFORM_PATH)/docker-ptf.dep
+include $(PLATFORM_PATH)/libsaithrift-dev.dep
 include $(PLATFORM_PATH)/one-image.dep
 include $(PLATFORM_PATH)/onie.dep
 include $(PLATFORM_PATH)/kvm-image.dep

platform/vs/rules.mk

@@ -3,6 +3,8 @@ include $(PLATFORM_PATH)/sonic-version.mk
 include $(PLATFORM_PATH)/docker-sonic-vs.mk
 include $(PLATFORM_PATH)/docker-syncd-vs.mk
 include $(PLATFORM_PATH)/docker-gbsyncd-vs.mk
+include $(PLATFORM_PATH)/docker-ptf.mk
+include $(PLATFORM_PATH)/libsaithrift-dev.mk
 include $(PLATFORM_PATH)/one-image.mk
 include $(PLATFORM_PATH)/onie.mk
 include $(PLATFORM_PATH)/kvm-image.mk

slave.mk

@@ -464,8 +464,8 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
 		if [ -f ./autogen.sh ]; then ./autogen.sh $(LOG); fi
 		$(SETUP_OVERLAYFS_FOR_DPKG_ADMINDIR)
 		$(if $($*_DPKG_TARGET),
-			DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) --admindir $$mergedir $(LOG),
-			DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $$mergedir $(LOG)
+			${$*_BUILD_ENV} DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --as-root -T$($*_DPKG_TARGET) --admindir $$mergedir $(LOG),
+			${$*_BUILD_ENV} DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC} ${$*_DEB_BUILD_OPTIONS}" dpkg-buildpackage -rfakeroot -b -us -uc -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $$mergedir $(LOG)
 		)
 		popd $(LOG_SIMPLE)
 		# Clean up