Rebase master

.azure-pipelines/official-build-cisco-8000.yml

@@ -22,13 +22,19 @@ resources:
     name: Cisco-8000-sonic/platform-cisco-8000
     endpoint: cisco-connection
 
+variables:
+- group: SONIC-AKV-STROAGE-1
+- name: StorageSASKey
+  value: $(sonicstorage-SasToken)
+
 stages:
 - stage: Build
   pool: sonic
   variables:
     CACHE_MODE: wcache
     SKIP_CHECKOUT: true
     TERM: ''
+    PACKAGE_URL: "https://sonicstorage.blob.core.windows.net/packages"
 
   jobs:
   - template: azure-pipelines-build.yml
@@ -60,5 +66,29 @@ stages:
           make PLATFORM=cisco-8000 platform/cisco-8000
           tar xfz $(System.ArtifactsDirectory)/artifactory-*.tar.gz -C platform/cisco-8000
         displayName: 'Setup cisco artifacts'
+      - script: |
+          set -ex
+          filename=$(find platform/cisco-8000/artifactory/sonic -name cisco-* -type f | head -n 1)
+          if [ -z "$filename" ]; then
+            echo "Cisco sai package not found" 1>&2
+            exit 1
+          fi
+          cd $(dirname $filename)
+          echo "PWD=$(pwd)"
+          ls -l *.deb
+          while read -r package; do
+            # Cisco version format: <VERSION>-sai-<sai-ver>-<distribution>-<COMMIT HASH>
+            # The <sai-ver> may contain several values in one build, the part is skipped when publishing to storage
+            # See https://github.com/Cisco-8000-sonic/sdk/blob/master/azure-pipelines.yml
+            # The $PACKAGE_URL is only accessible for AZP
+            version=$(echo $package | awk -F_ '{print $(NF-1)}' | cut -d- -f1,2,4,5)
+            package_url="$PACKAGE_URL/sai/ciscosai/master/$version/$package"
+            echo "Override package $package from $package_url"
+            wget "$package_url$StorageSASKey" -O "$package"
+          done < <(ls *.deb)
+        env:
+          StorageSASKey: $(StorageSASKey)
+        condition: ne(variables['Build.Reason'], 'PullRequest')
+        displayName: "Override cisco sai packages"
       jobGroups:
       - name: cisco-8000

.azure-pipelines/run-test-template.yml

@@ -7,6 +7,9 @@ parameters:
   type: string
 - name: ptf_name
   type: string
+- name: section
+  type: string
+  default: ''
 - name: image
   type: string
   default: sonic-vs.img.gz
@@ -43,7 +46,7 @@ steps:
 
 - script: |
     rm -rf $(Build.ArtifactStagingDirectory)/*
-    docker exec sonic-mgmt bash -c "/data/sonic-mgmt/tests/kvmtest.sh -en -T ${{ parameters.tbtype }} ${{ parameters.tbname }} ${{ parameters.dut }}"
+    docker exec sonic-mgmt bash -c "/data/sonic-mgmt/tests/kvmtest.sh -en -T ${{ parameters.tbtype }} ${{ parameters.tbname }} ${{ parameters.dut }} ${{ parameters.section }}"
   displayName: "Run tests"
 
 - script: |
@@ -75,17 +78,17 @@ steps:
   condition: succeededOrFailed()
 
 - publish: $(Build.ArtifactStagingDirectory)/kvmdump
-  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype}}.memdump@$(System.JobAttempt)
+  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype}}${{ parameters.section }}.memdump@$(System.JobAttempt)
   displayName: "Archive sonic kvm memdump"
   condition: failed()
 
 - publish: $(Build.ArtifactStagingDirectory)/logs
-  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype }}.log@$(System.JobAttempt)
+  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype }}${{ parameters.section }}.log@$(System.JobAttempt)
   displayName: "Archive sonic kvm logs"
   condition: succeededOrFailed()
 
 - task: PublishTestResults@2
   inputs:
     testResultsFiles: '$(Build.ArtifactStagingDirectory)/logs/**/*.xml'
-    testRunTitle: kvmtest.${{ parameters.tbtype }}
+    testRunTitle: kvmtest.${{ parameters.tbtype }}${{ parameters.section }}
   condition: succeededOrFailed()

.github/pull_request_template.md

@@ -30,13 +30,20 @@
 - [ ] 202006
 - [ ] 202012
 - [ ] 202106
+- [ ] 202111
 
 #### Description for the changelog
 <!--
 Write a short (one line) summary that describes the changes in this
 pull request for inclusion in the changelog:
 -->
 
+#### Link to config_db schema for YANG module changes
+<!--
+Provide a link to config_db schema for the table for which YANG model
+is defined
+Link should point to correct section on https://github.com/Azure/SONiC/wiki/Configuration.
+-->
 
 #### A picture of a cute animal (not mandatory but encouraged)
 

.gitmodules

@@ -99,4 +99,7 @@
 	url = https://github.com/Azure/sonic-linkmgrd.git
 [submodule "src/sonic-p4rt/sonic-pins"]
 	path = src/sonic-p4rt/sonic-pins
-	url = https://github.com/Azure/sonic-pins.git
+	url = https://github.com/Azure/sonic-pins.git
+[submodule "src/thrift_0_14_1/thrift"]
+	path = src/thrift_0_14_1/thrift
+	url = https://github.com/apache/thrift.git

Makefile.work

@@ -115,6 +115,10 @@ rules/config.user:
 include rules/config
 -include rules/config.user
 
+ifneq ($(DEFAULT_CONTAINER_REGISTRY),)
+override DEFAULT_CONTAINER_REGISTRY := $(DEFAULT_CONTAINER_REGISTRY)/
+endif
+
 ifeq ($(ENABLE_DOCKER_BASE_PULL),)
 	override ENABLE_DOCKER_BASE_PULL = n
 endif
@@ -139,9 +143,9 @@ $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
     scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
-$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
+$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
-$(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
+$(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
 
 # Add the versions in the tag, if the version change, need to rebuild the slave
 SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* src/sonic-build-hooks/hooks/* | sha1sum | awk '{print substr($$1,0,11);}')
@@ -270,6 +274,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_ENABLE_PFCWD_ON_START=$(ENABLE_PFCWD_ON_START) \
                            SONIC_ENABLE_SYNCD_RPC=$(ENABLE_SYNCD_RPC) \
                            SONIC_INSTALL_DEBUG_TOOLS=$(INSTALL_DEBUG_TOOLS) \
+                           SONIC_SAITHRIFT_V2=$(SAITHRIFT_V2) \
                            MDEBUG=$(MDEBUG) \
                            PASSWORD=$(PASSWORD) \
                            USERNAME=$(USERNAME) \
@@ -290,10 +295,12 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
+                           SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
                            ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \
                            ENABLE_AUTO_TECH_SUPPORT=$(ENABLE_AUTO_TECH_SUPPORT) \
                            BUILD_MULTIASIC_KVM=$(BUILD_MULTIASIC_KVM) \
+                           ENABLE_ASAN=$(ENABLE_ASAN) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset

azure-pipelines.yml

@@ -104,9 +104,9 @@ stages:
         testResultsFiles: '**/tr.xml'
         testRunTitle: vstest
 
-  - job:
+  - job: t0_part1
     pool: sonictest
-    displayName: "kvmtest-t0"
+    displayName: "kvmtest-t0-part1"
     timeoutInMinutes: 360
 
     steps:
@@ -116,6 +116,43 @@ stages:
         tbname: vms-kvm-t0
         ptf_name: ptf_vms6-1
         tbtype: t0
+        section: part-1
+
+  - job: t0_part2
+    pool: sonictest
+    displayName: "kvmtest-t0-part2"
+    timeoutInMinutes: 360
+
+    steps:
+    - template: .azure-pipelines/run-test-template.yml
+      parameters:
+        dut: vlab-01
+        tbname: vms-kvm-t0
+        ptf_name: ptf_vms6-1
+        tbtype: t0
+        section: part-2
+
+  - job:
+    pool: sonictest
+    displayName: "kvmtest-t0"
+    timeoutInMinutes: 360
+    dependsOn:
+    - t0_part1
+    - t0_part2
+    condition: always()
+    variables:
+      resultOfPart1: $[ dependencies.t0_part1.result ]
+      resultOfPart2: $[ dependencies.t0_part2.result ]
+
+    steps:
+    - script: |
+        if [ $(resultOfPart1) == "Succeeded" ] && [ $(resultOfPart2) == "Succeeded" ]; then
+          echo "Both job kvmtest-t0-part1 and kvmtest-t0-part2 are passed."
+          exit 0
+        else
+          echo "Either job kvmtest-t0-part1 or job kvmtest-t0-part2 failed! Please check the detailed information."
+          exit 1
+        fi
 
   - job:
     pool: sonictest-t1-lag

build_debian.sh

@@ -67,6 +67,9 @@ mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR
 mkdir -p $FILESYSTEM_ROOT/$PLATFORM_DIR/x86_64-grub
 touch $FILESYSTEM_ROOT/$PLATFORM_DIR/firsttime
 
+## ensure proc is mounted
+sudo mount proc /proc -t proc || true
+
 ## make / as a mountpoint in chroot env, needed by dockerd
 pushd $FILESYSTEM_ROOT
 sudo mount --bind . .
@@ -190,7 +193,7 @@ if [ -f platform/$CONFIGURED_PLATFORM/modules ]; then
 fi
 
 ## Add mtd and uboot firmware tools package
-sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install u-boot-tools mtd-utils device-tree-compiler
+sudo LANG=C chroot $FILESYSTEM_ROOT apt-get -y install u-boot-tools libubootenv-tool mtd-utils device-tree-compiler
 
 ## Install docker
 echo '[INFO] Install docker'
@@ -328,8 +331,13 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in
     jq                      \
     auditd
 
-# Change auditd log file path to fix auditd can't startup issue.
-sudo LANG=C chroot $FILESYSTEM_ROOT /bin/bash -c "sudo sed -i 's/^\s*log_file\s*=.*/log_file = \/var\/log\/audit.log/g' /etc/audit/auditd.conf"
+# Have systemd create the auditd log directory
+sudo mkdir -p ${FILESYSTEM_ROOT}/etc/systemd/system/auditd.service.d
+sudo tee ${FILESYSTEM_ROOT}/etc/systemd/system/auditd.service.d/log-directory.conf >/dev/null <<EOF
+[Service]
+LogsDirectory=audit
+LogsDirectoryMode=0750
+EOF
 
 if [[ $CONFIGURED_ARCH == amd64 ]]; then
 ## Pre-install the fundamental packages for amd64 (x86)
@@ -554,6 +562,9 @@ if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
     fi
 fi
 
+# Collect host image version files before cleanup
+scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
+
 # Remove GCC
 sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y remove gcc
 
@@ -566,6 +577,9 @@ sudo LANG=C chroot $FILESYSTEM_ROOT bash -c 'rm -rf /usr/share/doc/* /usr/share/
 ## Clean up proxy
 [ -n "$http_proxy" ] && sudo rm -f $FILESYSTEM_ROOT/etc/apt/apt.conf.d/01proxy
 
+## Clean up pip cache
+sudo LANG=C chroot $FILESYSTEM_ROOT pip3 cache purge
+
 ## Umount all
 echo '[INFO] Umount all'
 ## Display all process details access /proc
@@ -585,7 +599,6 @@ sudo rm -f $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS
 ## Note: -x to skip directories on different file systems, such as /proc
 sudo du -hsx $FILESYSTEM_ROOT
 sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker
-scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
 sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -comp zstd -b 1M -e boot -e var/lib/docker -e $PLATFORM_DIR
 
 # Ensure admin gid is 1000

build_image.sh

@@ -68,6 +68,8 @@ generate_kvm_image()
 
 generate_onie_installer_image()
 {
+    output_file=$OUTPUT_ONIE_IMAGE
+    [ -n "$1" ] && output_file=$1
     # Copy platform-specific ONIE installer config files where onie-mk-demo.sh expects them
     rm -rf ./installer/x86_64/platforms/
     mkdir -p ./installer/x86_64/platforms/
@@ -83,7 +85,7 @@ generate_onie_installer_image()
     ## Generate an ONIE installer image
     ## Note: Don't leave blank between lines. It is single line command.
     ./onie-mk-demo.sh $TARGET_PLATFORM $TARGET_MACHINE $TARGET_PLATFORM-$TARGET_MACHINE-$ONIEIMAGE_VERSION \
-          installer platform/$TARGET_MACHINE/platform.conf $OUTPUT_ONIE_IMAGE OS $IMAGE_VERSION $ONIE_IMAGE_PART_SIZE \
+          installer platform/$TARGET_MACHINE/platform.conf $output_file OS $IMAGE_VERSION $ONIE_IMAGE_PART_SIZE \
           $ONIE_INSTALLER_PAYLOAD
 }
 
@@ -119,12 +121,13 @@ if [ "$IMAGE_TYPE" = "onie" ]; then
 elif [ "$IMAGE_TYPE" = "raw" ]; then
 
     echo "Build RAW image"
+    tmp_output_onie_image=${OUTPUT_ONIE_IMAGE}.tmp
     mkdir -p `dirname $OUTPUT_RAW_IMAGE`
     sudo rm -f $OUTPUT_RAW_IMAGE
 
     generate_device_list "./installer/$TARGET_PLATFORM/platforms_asic"
 
-    generate_onie_installer_image
+    generate_onie_installer_image "$tmp_output_onie_image"
 
     echo "Creating SONiC raw partition : $OUTPUT_RAW_IMAGE of size $RAW_IMAGE_DISK_SIZE MB"
     fallocate -l "$RAW_IMAGE_DISK_SIZE"M $OUTPUT_RAW_IMAGE
@@ -135,8 +138,9 @@ elif [ "$IMAGE_TYPE" = "raw" ]; then
     ## Generate a partition dump that can be used to 'dd' in-lieu of using the onie-nos-installer
     ## Run the installer
     ## The 'build' install mode of the installer is used to generate this dump.
-    sudo chmod a+x $OUTPUT_ONIE_IMAGE
-    sudo ./$OUTPUT_ONIE_IMAGE
+    sudo chmod a+x $tmp_output_onie_image
+    sudo ./$tmp_output_onie_image
+    rm $tmp_output_onie_image
 
     [ -r $OUTPUT_RAW_IMAGE ] || {
         echo "Error : $OUTPUT_RAW_IMAGE not generated!"

device/accton/x86_64-accton_as4630_54pe-r0/installer.conf

@@ -1,3 +1,4 @@
 CONSOLE_PORT=0x3f8
 CONSOLE_DEV=0
 CONSOLE_SPEED=115200
+ONIE_PLATFORM_EXTRA_CMDLINE_LINUX="intel_iommu=off modprobe.blacklist=i2c-ismt,i2c_ismt,i2c-i801,i2c_i801"