Merge branch 'sonic-net:master' into ssd_util_plugin

Author: roger530-ho

.azure-pipelines/azure-pipelines-build.yml

@@ -121,6 +121,7 @@ jobs:
               mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz
             fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+            make $BUILD_OPTIONS target/docker-ptf-sai.gz
             if [ $(Build.Reason) != 'PullRequest' ];then
               gzip -kd target/sonic-vs.img.gz
               SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
@@ -138,7 +139,21 @@ jobs:
               make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
             fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
+              # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+              pushd ./src/sonic-sairedis/SAI
+              git stash
+              popd
+              if [ $(GROUP_NAME) == broadcom ]; then
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-dnx-rpc.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd 
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd
+              fi
             fi
             if [ $(syncd_rpc_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/sonic-$(GROUP_NAME).bin

.azure-pipelines/azure-pipelines-image-template.yml

@@ -36,7 +36,8 @@ jobs:
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
-        condition: eq(variables.SKIP_CHECKOUT, '') 
+        fetchDepth: 0
+        condition: and(succeeded(), eq(variables.SKIP_CHECKOUT, ''))
         displayName: 'Checkout code'
       - script: |
           BRANCH_NAME=$(Build.SourceBranchName)

.azure-pipelines/azure-pipelines-repd-build-variables.yml

@@ -1,6 +1,6 @@
 variables:
   ${{ if eq(variables['Build.Reason'],'PullRequest') }}:
-    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=$([ "$(System.PullRequest.TargetBranch)" != "master" ] && echo deb,py2,py3,web,git,docker)'
+    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=$([[ "$(System.PullRequest.TargetBranch)" =~ ^20[2-9][0-9]{3}$ ]] && echo deb,py2,py3,web,git,docker)'
   ${{ elseif ne(variables['Build.SourceBranchName'],'master') }}:
     VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
   ${{ else }}:

.azure-pipelines/build-template.yml

@@ -92,6 +92,7 @@ jobs:
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+        make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/docker-ptf-sai.gz
       else
         if [ ${{ parameters.dbg_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-${{ parameters.platform }}.bin && \
@@ -105,6 +106,16 @@ jobs:
         fi
         if [ ${{ parameters.sync_rpc_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
+          # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache
+          pushd ./src/sonic-sairedis/SAI
+          git stash
+          popd 
+          if [ ${{ parameters.platform }} == broadcom ]; then
+            make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+            pushd ./src/sonic-sairedis/SAI
+            git stash
+            popd 
+          fi
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin

.azure-pipelines/docker-sonic-slave-arm64.yml

@@ -10,6 +10,9 @@ parameters:
   - amd64
   - armhf
   - arm64
+- name: march
+  type: string
+  default: ''
 - name: dist
   type: string
   values:
@@ -32,89 +35,44 @@ parameters:
   - sonicbld-armhf
 
 jobs:
-- job: Build_${{ parameters.dist }}_${{ parameters.arch }}
+- job: Build_${{ parameters.dist }}_${{ parameters.march }}${{ parameters.arch }}
   timeoutInMinutes: 360
+  variables:
+  - template: /.azure-pipelines/template-variables.yml@buildimage
+  - template: /.azure-pipelines/azure-pipelines-repd-build-variables.yml@buildimage
   pool: ${{ parameters.pool }}
   steps:
   - template: cleanup.yml
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-    - template: template-clean-sonic-slave.yml
-  - ${{ else }}:
-    - template: '/.azure-pipelines/template-clean-sonic-slave.yml@buildimage'
+  - template: /.azure-pipelines/template-clean-sonic-slave.yml@buildimage
   - checkout: self
     clean: true
     submodules: recursive
+  - task: Docker@2
+    displayName: Login to ACR
+    inputs:
+      command: login
+      containerRegistry: ${{ parameters.registry_conn }}
   - bash: |
       set -ex
+      image_tag=$(BLDENV=${{ parameters.dist }} make -f Makefile.work showtag PLATFORM=generic PLATFORM_ARCH=${{ parameters.arch }} | grep sonic-slave | tail -n 1)
+      image_latest=$(echo $(echo $image_tag | awk -F: '{print$1}'):latest)
+      docker rmi $image_tag || true
 
-      SLAVE_DIR=sonic-slave-${{ parameters.dist }}
-      if [ x${{ parameters.pool }} == x"sonicbld" ]; then
-        if [ x${{ parameters.arch }} == x"amd64" ]; then
-          SLAVE_BASE_IMAGE=${SLAVE_DIR}
-          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}
-        elif [ x${{ parameters.pool }} == x"sonicbld" ]; then
-          SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ parameters.arch }}
-          SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-march-${{ parameters.arch }}
-        fi
-      elif [[ x${{ parameters.pool }} == x"sonicbld-armhf" && x${{ parameters.arch }} == x"armhf" ]]; then
-        SLAVE_BASE_IMAGE=${SLAVE_DIR}
-        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-armhf
-      elif [[ x${{ parameters.pool }} == x"sonicbld-arm64" && x${{ parameters.arch }} == x"arm64" ]]; then
-        SLAVE_BASE_IMAGE=${SLAVE_DIR}
-        SLAVE_BASE_IMAGE_UPLOAD=${SLAVE_DIR}-arm64
-      else
-        echo "do not support build ${{ parameters.arch }} on ${{ parameters.pool }}"
-        exit 1
+      if [[ "$(Build.Reason)" =~ [a-zA-Z]*CI ]] && docker pull ${{ parameters.registry_url }}/${image_tag};then
+        exit 0
       fi
 
-      if [ x"$(Build.SourceBranchName)" == x"202012" ]; then
-        BUILD_OPTIONS = 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
+      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ parameters.dist }} make -f Makefile.work configure PLATFORM=generic PLATFORM_ARCH=${{ parameters.arch }} $args || docker image ls $image_tag
+      if [[ "$(Build.Reason)" == "PullRequest" ]];then
+        exit 0
       fi
 
-      tmpfile=$(mktemp)
-
-      echo ${{ parameters.arch }} > .arch
-
-      DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ parameters.dist }} $(BUILD_OPTIONS) make -f Makefile.work sonic-slave-build | tee $tmpfile
-      SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
-      SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
-
-      mkdir -p target
-
-      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:latest
-      docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:$SLAVE_BASE_TAG
-      if [ "$SLAVE_BASE_IMAGE_UPLOAD" != "$SLAVE_DIR" ]; then
-        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:latest-${{ parameters.arch }}
-        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:$SLAVE_BASE_TAG
+      docker tag ${image_tag} ${REGISTRY_SERVER}/${image_tag}
+      docker push ${REGISTRY_SERVER}/${image_tag}
+      if [[ "${{ parameters.arch }}" == "amd64" ]];then
+        docker tag ${image_tag} ${REGISTRY_SERVER}/${image_latest}
+        docker push ${REGISTRY_SERVER}/${image_latest}
       fi
-      set +x
-      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE_UPLOAD"
-      echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
     env:
       REGISTRY_SERVER: ${{ parameters.registry_url }}
     displayName: Build sonic-slave-${{ parameters.dist }}-${{ parameters.arch }}
-
-  - task: Docker@2
-    condition: ne(variables['Build.Reason'], 'PullRequest')
-    displayName: Upload image
-    inputs:
-      containerRegistry: ${{ parameters.registry_conn }}
-      repository: $(VARIABLE_SLAVE_BASE_IMAGE)
-      command: push
-      ${{ if eq(variables['Build.SourceBranchName'], 'master') }}:
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG)
-          latest
-      ${{ else }}:
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG)
-  - ${{ if ne(parameters.arch, 'amd64') }}:
-    - task: Docker@2
-      condition: ne(variables['Build.Reason'], 'PullRequest')
-      displayName: Upload image ${{ parameters.dist }}
-      inputs:
-        containerRegistry: ${{ parameters.registry_conn }}
-        repository: "sonic-slave-${{ parameters.dist }}"
-        command: push
-        tags: |
-          $(VARIABLE_SLAVE_BASE_TAG) 

.azure-pipelines/docker-sonic-slave-armhf.yml

@@ -12,26 +12,27 @@ resources:
     endpoint: sonic-net
 
 schedules:
-- cron: "0 8 * * *"
+- cron: "0 0 * * 0"
+  displayName: Weekly build
   branches:
     include:
     - master
-    - 202012
+    - 202???
   always: true
 
-trigger: none
-pr:
+pr: none
+trigger:
+  batch: true
   branches:
     include:
     - master
+    - 202???
   paths:
     include:
-    - sonic-slave-jessie
-    - sonic-slave-stretch
-    - sonic-slave-buster
-    - sonic-slave-bullseye
-    - src/sonic-build-hooks
-    - .azure-pipelines
+    - sonic-slave-*
+    - files/build/versions
+    - Makefile
+    - Makefile.work
 
 parameters:
 - name: 'arches'
@@ -60,15 +61,21 @@ stages:
   - ${{ each dist in parameters.dists }}:
     - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
       - ${{ each arch in parameters.arches }}:
-        - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-          - template: docker-sonic-slave-template.yml
-            parameters:
-              pool: sonicbld
-              arch: ${{ arch }}
-              dist: ${{ dist }}
-        - ${{ else }}:
-          - template: '/.azure-pipelines/docker-sonic-slave-template.yml@buildimage'
+        - template: .azure-pipelines/docker-sonic-slave-template.yml@buildimage
+          parameters:
+            pool: sonicbld
+            arch: ${{ arch }}
+            dist: ${{ dist }}
+- stage: Build_march
+  dependsOn: []
+  jobs:
+  - ${{ each dist in parameters.dists }}:
+    - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
+      - ${{ each arch in parameters.arches }}:
+        - ${{ if ne(arch, 'amd64') }}:
+          - template: .azure-pipelines/docker-sonic-slave-template.yml@buildimage
             parameters:
-              pool: sonicbld
+              pool: sonicbld-${{ arch }}
               arch: ${{ arch }}
               dist: ${{ dist }}
+              march: march_