diff --git a/.azure-pipelines/run-test-scheduler-template.yml b/.azure-pipelines/run-test-scheduler-template.yml new file mode 100644 index 00000000000..41956381fba --- /dev/null +++ b/.azure-pipelines/run-test-scheduler-template.yml @@ -0,0 +1,114 @@ +parameters: +- name: TOPOLOGY + type: string + +- name: POLL_INTERVAL + type: number + default: 10 + +- name: POLL_TIMEOUT + type: number + default: 36000 + +- name: MIN_WORKER + type: number + default: 1 + +- name: MAX_WORKER + type: number + default: 2 + +- name: TEST_SET + type: string + default: "" + +- name: DEPLOY_MG_EXTRA_PARAMS + type: string + default: "" + +steps: + - script: | + set -ex + wget -O ./.azure-pipelines/test_plan.py https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py + wget -O ./.azure-pipelines/pr_test_scripts.yaml https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/pr_test_scripts.yaml + displayName: Download TestbedV2 scripts + + - script: | + set -ex + pip install PyYAML + rm -f new_test_plan_id.txt + python ./.azure-pipelines/test_plan.py create -t ${{ parameters.TOPOLOGY }} -o new_test_plan_id.txt --min-worker ${{ parameters.MIN_WORKER }} --max-worker ${{ parameters.MAX_WORKER }} --test-set ${{ parameters.TEST_SET }} --kvm-build-id $(KVM_BUILD_ID) --deploy-mg-extra-params "${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" + TEST_PLAN_ID=`cat new_test_plan_id.txt` + + echo "Created test plan $TEST_PLAN_ID" + echo "Check https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID for test plan status" + echo "##vso[task.setvariable variable=TEST_PLAN_ID]$TEST_PLAN_ID" + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + TENANT_ID: $(TESTBED_TOOLS_MSAL_TENANT_ID) + CLIENT_ID: $(TESTBED_TOOLS_MSAL_CLIENT_ID) + CLIENT_SECRET: $(TESTBED_TOOLS_MSAL_CLIENT_SECRET) + displayName: Trigger test + + - script: | + set -ex + echo "Lock testbed" + echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com" + echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID" + # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED" + python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states PREPARE_TESTBED EXECUTING KVMDUMP FINISHED CANCELLED FAILED + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + displayName: Lock testbed + timeoutInMinutes: 240 + + - script: | + set -ex + echo "Prepare testbed" + echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient(We will improve the indication in a short time)" + echo "If the progress keeps as 0 for more than 1 hour, please cancel and retry this pipeline" + echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com" + echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID" + # When "PREPARE_TESTBED" finish, it changes into "EXECUTING" + python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 2400 --expected-states EXECUTING KVMDUMP FINISHED CANCELLED FAILED + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + displayName: Prepare testbed + timeoutInMinutes: 40 + + - script: | + set -ex + echo "Run test" + echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com" + echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID" + # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED" + python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 18000 --expected-states KVMDUMP FINISHED CANCELLED FAILED + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + displayName: Run test + timeoutInMinutes: 300 + + - script: | + set -ex + echo "KVM dump" + echo "TestbedV2 is just online and might not be stable enough, for any issue, please send email to sonictestbedtools@microsoft.com" + echo "Runtime detailed progress at https://www.testbed-tools.org/scheduler/testplan/$TEST_PLAN_ID" + # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED" + python ./.azure-pipelines/test_plan.py poll -i "$(TEST_PLAN_ID)" --timeout 43200 --expected-states FINISHED CANCELLED FAILED + condition: always() + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + displayName: KVM dump + timeoutInMinutes: 20 + + - script: | + set -ex + echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect." + python ./.azure-pipelines/test_plan.py cancel -i "$(TEST_PLAN_ID)" + condition: always() + env: + TESTBED_TOOLS_URL: $(TESTBED_TOOLS_URL) + TENANT_ID: $(TESTBED_TOOLS_MSAL_TENANT_ID) + CLIENT_ID: $(TESTBED_TOOLS_MSAL_CLIENT_ID) + CLIENT_SECRET: $(TESTBED_TOOLS_MSAL_CLIENT_SECRET) + displayName: Finalize running test plan diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a9004af559e..74caad8f007 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,6 +76,7 @@ stages: dependsOn: BuildVS condition: and(ne(stageDependencies.BuildVS.outputs['vs.SetVar.SKIP_VSTEST'], 'YES'), in(dependencies.BuildVS.result, 'Succeeded', 'SucceededWithIssues')) variables: + - group: Testbed-Tools - name: inventory value: veos_vtb - name: testbed_file @@ -138,7 +139,8 @@ stages: - job: t0_part1 pool: sonictest displayName: "kvmtest-t0-part1" - timeoutInMinutes: 360 + timeoutInMinutes: 400 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES')) continueOnError: false steps: - template: .azure-pipelines/run-test-template.yml @@ -147,12 +149,14 @@ stages: tbname: vms-kvm-t0 ptf_name: ptf_vms6-1 tbtype: t0 + vmtype: ceos section: part-1 - job: t0_part2 pool: sonictest displayName: "kvmtest-t0-part2" - timeoutInMinutes: 360 + timeoutInMinutes: 400 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES')) continueOnError: false steps: - template: .azure-pipelines/run-test-template.yml @@ -161,35 +165,76 @@ stages: tbname: vms-kvm-t0 ptf_name: ptf_vms6-1 tbtype: t0 + vmtype: ceos section: part-2 + - job: t0_testbedv2 + pool: + vmImage: 'ubuntu-20.04' + displayName: "kvmtest-t0 by TestbedV2" + timeoutInMinutes: 1080 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES')) + continueOnError: false + steps: + - template: .azure-pipelines/run-test-scheduler-template.yml + parameters: + TOPOLOGY: t0 + MIN_WORKER: 2 + MAX_WORKER: 3 + + - job: t0_2vlans_testbedv2 + pool: + vmImage: 'ubuntu-20.04' + displayName: "kvmtest-t0-2vlans by TestbedV2" + timeoutInMinutes: 1080 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES')) + continueOnError: false + steps: + - template: .azure-pipelines/run-test-scheduler-template.yml + parameters: + TOPOLOGY: t0 + TEST_SET: t0-2vlans + MAX_WORKER: 1 + DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a" + - job: - pool: sonictest + pool: + vmImage: 'ubuntu-20.04' displayName: "kvmtest-t0" - timeoutInMinutes: 360 dependsOn: - t0_part1 - t0_part2 + - t0_testbedv2 + - t0_2vlans_testbedv2 condition: always() continueOnError: false variables: resultOfPart1: $[ dependencies.t0_part1.result ] resultOfPart2: $[ dependencies.t0_part2.result ] + resultOfT0TestbedV2: $[ dependencies.t0_testbedv2.result ] + resultOfT02VlansTestbedV2: $[ dependencies.t0_2vlans_testbedv2.result ] steps: - script: | + if [ $(resultOfT0TestbedV2) == "Succeeded" ] && [ $(resultOfT02VlansTestbedV2) == "Succeeded" ]; then + echo "TestbedV2 t0 passed." + exit 0 + fi + if [ $(resultOfPart1) == "Succeeded" ] && [ $(resultOfPart2) == "Succeeded" ]; then - echo "Both job kvmtest-t0-part1 and kvmtest-t0-part2 are passed." + echo "Classic t0 jobs(both part1 and part2) passed." exit 0 - else - echo "Either job kvmtest-t0-part1 or job kvmtest-t0-part2 failed! Please check the detailed information." - exit 1 fi - - job: + echo "Both classic and TestbedV2 t0 jobs failed! Please check the detailed information. (Any of them passed, t0 will be considered as passed)" + exit 1 + + + - job: t1_lag_classic pool: sonictest-t1-lag - displayName: "kvmtest-t1-lag" - timeoutInMinutes: 360 + displayName: "kvmtest-t1-lag classic" + timeoutInMinutes: 400 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_CLASSICAL_TEST, 'YES')) continueOnError: false steps: - template: .azure-pipelines/run-test-template.yml @@ -198,6 +243,43 @@ stages: tbname: vms-kvm-t1-lag ptf_name: ptf_vms6-2 tbtype: t1-lag + vmtype: ceos + + - job: t1_lag_testbedv2 + pool: + vmImage: 'ubuntu-20.04' + displayName: "kvmtest-t1-lag by TestbedV2" + timeoutInMinutes: 600 + condition: and(succeeded(), eq(variables.BUILD_IMG_RUN_TESTBEDV2_TEST, 'YES')) + continueOnError: false + steps: + - template: .azure-pipelines/run-test-scheduler-template.yml + parameters: + TOPOLOGY: t1-lag + MIN_WORKER: 2 + MAX_WORKER: 3 + + - job: + pool: + vmImage: 'ubuntu-20.04' + displayName: "kvmtest-t1-lag" + dependsOn: + - t1_lag_classic + - t1_lag_testbedv2 + condition: always() + continueOnError: false + variables: + resultOfClassic: $[ dependencies.t1_lag_classic.result ] + resultOfTestbedV2: $[ dependencies.t1_lag_testbedv2.result ] + steps: + - script: | + if [ $(resultOfClassic) == "Succeeded" ] || [ $(resultOfTestbedV2) == "Succeeded" ]; then + echo "One or both of t1_lag_classic and t1_lag_testbedv2 passed." + exit 0 + else + echo "Both t1_lag_classic and t1_lag_testbedv2 failed! Please check the detailed information." + exit 1 + fi - job: pool: sonictest-sonic-t0