diff --git a/.azure-pipelines/baseline_test/baseline.test.mgmt.public.yml b/.azure-pipelines/baseline_test/baseline.test.mgmt.public.yml index 739ba4eda0e..91b745f03df 100644 --- a/.azure-pipelines/baseline_test/baseline.test.mgmt.public.yml +++ b/.azure-pipelines/baseline_test/baseline.test.mgmt.public.yml @@ -11,53 +11,78 @@ schedules: - master always: true -stages: +parameters: + - name: BUILD_REASON + type: string + default: "BaselineTest" -- stage: Test_round_1 - variables: + - name: TEST_PLAN_NUM + type: string + default: $(BASELINE_MGMT_PUBLIC_MASTER_TEST_NUM) + + - name: RETRY_TIMES + type: string + default: "0" + + - name: TEST_PLAN_STOP_ON_FAILURE + type: string + default: "False" + +variables: - group: SONiC-Elastictest - name: inventory value: veos_vtb - name: testbed_file value: vtestbed.yaml + - name: BUILD_BRANCH + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + value: $(System.PullRequest.TargetBranch) + ${{ else }}: + value: $(Build.SourceBranchName) + +stages: + +- stage: Test_round_1 jobs: - - template: baseline.test.template.yml + - template: ../pr_test_template.yml + - parameters: + BUILD_REASON: ${{ parameters.BUILD_REASON }} + TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} + RETRY_TIMES: ${{ parameters.RETRY_TIMES }} + TEST_PLAN_STOP_ON_FAILURE: ${{ parameters.TEST_PLAN_STOP_ON_FAILURE }} - stage: Test_round_2 dependsOn: - Test_round_1 condition: succeededOrFailed('Test_round_1') - variables: - - group: SONiC-Elastictest - - name: inventory - value: veos_vtb - - name: testbed_file - value: vtestbed.yaml jobs: - - template: baseline.test.template.yml + - template: ../pr_test_template.yml + - parameters: + BUILD_REASON: ${{ parameters.BUILD_REASON }} + TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} + RETRY_TIMES: ${{ parameters.RETRY_TIMES }} + TEST_PLAN_STOP_ON_FAILURE: ${{ parameters.TEST_PLAN_STOP_ON_FAILURE }} - stage: Test_round_3 dependsOn: - Test_round_2 condition: succeededOrFailed('Test_round_2') - variables: - - group: SONiC-Elastictest - - name: inventory - value: veos_vtb - - name: testbed_file - value: vtestbed.yaml jobs: - - template: baseline.test.template.yml + - template: ../pr_test_template.yml + - parameters: + BUILD_REASON: ${{ parameters.BUILD_REASON }} + TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} + RETRY_TIMES: ${{ parameters.RETRY_TIMES }} + TEST_PLAN_STOP_ON_FAILURE: ${{ parameters.TEST_PLAN_STOP_ON_FAILURE }} - stage: Test_round_4 dependsOn: - Test_round_3 condition: succeededOrFailed('Test_round_3') - variables: - - group: SONiC-Elastictest - - name: inventory - value: veos_vtb - - name: testbed_file - value: vtestbed.yaml jobs: - - template: baseline.test.template.yml + - template: ../pr_test_template.yml + - parameters: + BUILD_REASON: ${{ parameters.BUILD_REASON }} + TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} + RETRY_TIMES: ${{ parameters.RETRY_TIMES }} + TEST_PLAN_STOP_ON_FAILURE: ${{ parameters.TEST_PLAN_STOP_ON_FAILURE }} diff --git a/.azure-pipelines/pr_test_template.yml b/.azure-pipelines/pr_test_template.yml new file mode 100644 index 00000000000..be8859e2033 --- /dev/null +++ b/.azure-pipelines/pr_test_template.yml @@ -0,0 +1,129 @@ +parameters: +- name: BUILD_REASON + type: string + default: "PullRequest" + +- name: TIMEOUT_IN_MINUTES_PR_TEST + type: number + default: 480 + +- name: MAX_RUN_TEST_MINUTES + type: number + default: 240 + +- name: TEST_PLAN_NUM + type: string + default: "1" + +- name: TEST_PLAN_STOP_ON_FAILURE + type: string + default: "True" + +- name: RETRY_TIMES + type: string + default: "2" + +jobs: + - job: get_impacted_area + cancelTimeoutInMinutes: 10 + displayName: "Get impacted area" + timeoutInMinutes: 240 + continueOnError: false + pool: sonic-ubuntu-1c + steps: + - template: impacted_area_testing/get-impacted-area.yml + + - job: impacted_area_kvmtest + displayName: " " + cancelTimeoutInMinutes: 10 + dependsOn: get_impacted_area + strategy: + matrix: + impacted-area-kvmtest-t0_by_Elastictest: + TESTBED_PREP_TOPOLOGY: t0 + CHECKER: t0_checker + TOPOLOGY: t0 + impacted-area-kvmtest-t0-2vlans_by_Elastictest: + TESTBED_PREP_TOPOLOGY: t0-2vlans + CHECKER: t0-2vlans_checker + DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a " + TOPOLOGY: t0 + impacted-area-kvmtest-t1-lag_by_Elastictest: + TESTBED_PREP_TOPOLOGY: t1 + CHECKER: t1_checker + TOPOLOGY: t1-lag + impacted-area-kvmtest-dualtor_by_Elastictest: + TESTBED_PREP_TOPOLOGY: dualtor + CHECKER: dualtor_checker + COMMON_EXTRA_PARAMS: "--disable_loganalyzer --disable_sai_validation " + TOPOLOGY: dualtor + impacted-area-kvmtest-multi-asic-t1_by_Elastictest: + TESTBED_PREP_TOPOLOGY: t1-multi-asic + CHECKER: t1-multi-asic_checker + TOPOLOGY: t1-8-lag + NUM_ASIC: 4 + impacted-area-kvmtest-multi-asic-t1_by_Elastictest_optional: + TESTBED_PREP_TOPOLOGY: t1 + CHECKER: t1_checker + TOPOLOGY: t1-8-lag + NUM_ASIC: 4 + STOP_ON_FAILURE: "False" + impacted-area-kvmtest-t0-sonic_by_Elastictest: + TESTBED_PREP_TOPOLOGY: t0-sonic + CHECKER: t0-sonic_checker + COMMON_EXTRA_PARAMS: "--neighbor_type=sonic --disable_sai_validation " + TOPOLOGY: t0-64-32 + VM_TYPE: vsonic + SPECIFIC_PARAM: '[ + {"name": "bgp/test_bgp_fact.py", "param": "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"}, + {"name": "macsec", "param": "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"} + ]' + impacted-area-kvmtest-dpu_by_Elastictest: + TESTBED_PREP_TOPOLOGY: dpu + CHECKER: dpu_checker + TOPOLOGY: dpu + SPECIFIC_PARAM: '[ + {"name": "dash/test_dash_vnet.py", "param": "--skip_dataplane_checking"} + ]' + impacted-area-kvmtest-t2_by_Elastictest_optional: + TESTBED_PREP_TOPOLOGY: t2 + CHECKER: t2_checker + TOPOLOGY: t2 + STOP_ON_FAILURE: "False" + + condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], variables['CHECKER']) + variables: + TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] + NUM_ASIC: 1 + VM_TYPE: ceos + STOP_ON_FAILURE: ${{ parameters.TEST_PLAN_STOP_ON_FAILURE }} + COMMON_EXTRA_PARAMS: "--disable_sai_validation " + DEPLOY_MG_EXTRA_PARAMS: "" + SPECIFIC_PARAM: "[]" + timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} + continueOnError: false + pool: sonic-ubuntu-1c + steps: + - template: impacted_area_testing/calculate-instance-numbers.yml + parameters: + TOPOLOGY: $(TESTBED_PREP_TOPOLOGY) + BUILD_BRANCH: $(BUILD_BRANCH) + + - template: run-test-elastictest-template.yml + parameters: + TOPOLOGY: $(TOPOLOGY) + SCRIPTS: $(SCRIPTS) + MIN_WORKER: $(INSTANCE_NUMBER) + MAX_WORKER: $(INSTANCE_NUMBER) + DEPLOY_MG_EXTRA_PARAMS: $(DEPLOY_MG_EXTRA_PARAMS) + COMMON_EXTRA_PARAMS: $(COMMON_EXTRA_PARAMS) + KVM_IMAGE_BRANCH: $(BUILD_BRANCH) + MGMT_BRANCH: $(BUILD_BRANCH) + BUILD_REASON: ${{ parameters.BUILD_REASON }} + RETRY_TIMES: ${{ parameters.RETRY_TIMES }} + STOP_ON_FAILURE: $(STOP_ON_FAILURE) + TEST_PLAN_NUM: ${{ parameters.TEST_PLAN_NUM }} + NUM_ASIC: $(NUM_ASIC) + VM_TYPE: $(VM_TYPE) + SPECIFIC_PARAM: $(SPECIFIC_PARAM) + MAX_RUN_TEST_MINUTES: ${{ parameters.MAX_RUN_TEST_MINUTES }} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 059a937b956..c9ba0746c02 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,10 +37,6 @@ resources: ref: master endpoint: sonic-net -parameters: -- name: TIMEOUT_IN_MINUTES_PR_TEST - type: number - default: 480 stages: - template: /.azure-pipelines/common2/stages/analyze-code.yml @@ -96,266 +92,4 @@ stages: value: $(Build.SourceBranchName) jobs: -# Below are Impacted Area Based PR checkers - - job: get_impacted_area - displayName: "Get impacted area" - timeoutInMinutes: 10 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/get-impacted-area.yml - parameters: - BUILD_BRANCH: $(BUILD_BRANCH) - - - job: impacted_area_t0_elastictest - displayName: "impacted-area-kvmtest-t0 by Elastictest" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: t0 - BUILD_BRANCH: $(BUILD_BRANCH) - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t0 - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - COMMON_EXTRA_PARAMS: "--disable_sai_validation " - - - job: impacted_area_t0_2vlans_elastictest - displayName: "impacted-area-kvmtest-t0-2vlans by Elastictest" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0-2vlans_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: t0-2vlans - BUILD_BRANCH: $(BUILD_BRANCH) - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t0 - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a" - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - COMMON_EXTRA_PARAMS: "--disable_sai_validation " - - - job: impacted_area_t1_lag_elastictest - displayName: "impacted-area-kvmtest-t1-lag by Elastictest" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: t1 - BUILD_BRANCH: $(BUILD_BRANCH) - # 40 mins for preparing testbed, 10 mins for pre-test and post-test - PREPARE_TIME: 50 - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t1-lag - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - COMMON_EXTRA_PARAMS: "--disable_sai_validation " - - - job: impacted_area_dualtor_elastictest - displayName: "impacted-area-kvmtest-dualtor by Elastictest" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 'dualtor_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: dualtor - BUILD_BRANCH: $(BUILD_BRANCH) - # 30 mins for preparing testbed, 30 mins for pre-test and 20 mins for post-test - PREPARE_TIME: 80 - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: dualtor - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - COMMON_EXTRA_PARAMS: "--disable_loganalyzer --disable_sai_validation " - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - - - job: impacted_area_multi_asic_elastictest - displayName: "impacted-area-kvmtest-multi-asic-t1 by Elastictest" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1-multi-asic_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} - continueOnError: true - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: t1-multi-asic - BUILD_BRANCH: $(BUILD_BRANCH) - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t1-8-lag - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - NUM_ASIC: 4 - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - COMMON_EXTRA_PARAMS: "--disable_sai_validation " - - - job: impacted_area_t0_sonic_elastictest - displayName: "impacted-area-kvmtest-t0-sonic by Elastictest" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't0-sonic_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: t0-sonic - BUILD_BRANCH: $(BUILD_BRANCH) - PREPARE_TIME: 40 - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t0-64-32 - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - COMMON_EXTRA_PARAMS: "--neighbor_type=sonic --disable_sai_validation " - VM_TYPE: vsonic - MGMT_BRANCH: $(BUILD_BRANCH) - SPECIFIC_PARAM: '[ - {"name": "bgp/test_bgp_fact.py", "param": "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"}, - {"name": "macsec", "param": "--neighbor_type=sonic --enable_macsec --macsec_profile=128_SCI,256_XPN_SCI"} - ]' - - - job: impacted_area_dpu_elastictest - displayName: "impacted-area-kvmtest-dpu by Elastictest" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 'dpu_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: dpu - BUILD_BRANCH: $(BUILD_BRANCH) - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: dpu - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - COMMON_EXTRA_PARAMS: "--disable_sai_validation " - SPECIFIC_PARAM: '[ - {"name": "dash/test_dash_vnet.py", "param": "--skip_dataplane_checking"} - ]' - - # This PR checker aims to run all t1 test scripts on multi-asic topology. - - job: impacted_area_multi_asic_t1_elastictest - displayName: "impacted-area-kvmtest-multi-asic-t1 by Elastictest - optional" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't1_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: ${{ parameters.TIMEOUT_IN_MINUTES_PR_TEST }} - continueOnError: true - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: t1 - BUILD_BRANCH: $(BUILD_BRANCH) - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t1-8-lag - STOP_ON_FAILURE: "False" - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - NUM_ASIC: 4 - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - COMMON_EXTRA_PARAMS: "--disable_sai_validation " - - - job: impacted_area_t2_elastictest - displayName: "impacted-area-kvmtest-t2 by Elastictest - optional" - dependsOn: - - get_impacted_area - condition: contains(dependencies.get_impacted_area.outputs['SetVariableTask.PR_CHECKERS'], 't2_checker') - variables: - TEST_SCRIPTS: $[ dependencies.get_impacted_area.outputs['SetVariableTask.TEST_SCRIPTS'] ] - timeoutInMinutes: 240 - continueOnError: true - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/impacted_area_testing/calculate-instance-numbers.yml - parameters: - TOPOLOGY: t2 - BUILD_BRANCH: $(BUILD_BRANCH) - - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t2 - STOP_ON_FAILURE: "False" - SCRIPTS: $(SCRIPTS) - MIN_WORKER: $(INSTANCE_NUMBER) - MAX_WORKER: $(INSTANCE_NUMBER) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - COMMON_EXTRA_PARAMS: "--disable_sai_validation" + - template: .azure-pipelines/pr_test_template.yml