diff --git a/.azure-pipelines/run-test-elastictest-template.yml b/.azure-pipelines/run-test-elastictest-template.yml index d95af465bc4..0060686d26b 100644 --- a/.azure-pipelines/run-test-elastictest-template.yml +++ b/.azure-pipelines/run-test-elastictest-template.yml @@ -171,149 +171,174 @@ steps: displayName: "Download test plan script" - script: | - set -e - - pip install PyYAML - - rm -f new_test_plan_id.txt - - python ./.azure-pipelines/test_plan.py create \ - -t ${{ parameters.TOPOLOGY }} \ - -o new_test_plan_id.txt \ - --min-worker ${{ parameters.MIN_WORKER }} \ - --max-worker ${{ parameters.MAX_WORKER }} \ - --lock-wait-timeout-seconds ${{ parameters.LOCK_WAIT_TIMEOUT_SECONDS }} \ - --test-set ${{ parameters.TEST_SET }} \ - --kvm-build-id $(KVM_BUILD_ID) \ - --kvm-image-branch "${{ parameters.KVM_IMAGE_BRANCH }}" \ - --deploy-mg-extra-params="${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" \ - --common-extra-params="${{ parameters.COMMON_EXTRA_PARAMS }}" \ - --vm-type ${{ parameters.VM_TYPE }} --num-asic ${{ parameters.NUM_ASIC }} \ - --image_url ${{ parameters.IMAGE_URL }} \ - --upgrade-image-param="${{ parameters.UPGRADE_IMAGE_PARAM }}" \ - --hwsku ${{ parameters.HWSKU }} \ - --test-plan-type ${{ parameters.TEST_PLAN_TYPE }} \ - --platform ${{ parameters.PLATFORM }} \ - --testbed-name "${{ parameters.TESTBED_NAME }}" \ - --scripts "${{ parameters.SCRIPTS }}" \ - --features "${{ parameters.FEATURES }}" \ - --scripts-exclude "${{ parameters.SCRIPTS_EXCLUDE }}" \ - --features-exclude "${{ parameters.FEATURES_EXCLUDE }}" \ - --specific-param='${{ parameters.SPECIFIC_PARAM }}' \ - --affinity='${{ parameters.AFFINITY }}' \ - --build-reason ${{ parameters.BUILD_REASON }} \ - --repo-name ${{ parameters.REPO_NAME }} \ - --mgmt-branch ${{ parameters.MGMT_BRANCH }} \ - --stop-on-failure ${{ parameters.STOP_ON_FAILURE }} \ - --retry-times ${{ parameters.RETRY_TIMES }} \ - --dump-kvm-if-fail ${{ parameters.DUMP_KVM_IF_FAIL }} \ - --requester "${{ parameters.REQUESTER }}" \ - --max-execute-seconds $((${{ parameters.MAX_RUN_TEST_MINUTES }} * 60)) \ - --test-plan-num ${{ parameters.TEST_PLAN_NUM }} - - TEST_PLAN_ID_LIST=( $(cat new_test_plan_id.txt) ) - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo "Created test plan $TEST_PLAN_ID" - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - done - TEST_PLAN_ID_LIST_STRING=$(printf "%s," "${TEST_PLAN_ID_LIST[@]}") - TEST_PLAN_ID_LIST_STRING=${TEST_PLAN_ID_LIST_STRING%,} - echo "##vso[task.setvariable variable=TEST_PLAN_ID_LIST_STRING]$TEST_PLAN_ID_LIST_STRING" + # Ensure that azure-cli apt package is installed + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + displayName: "Install azure-cli" + + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -e + + pip install PyYAML + + rm -f new_test_plan_id.txt + + python ./.azure-pipelines/test_plan.py create \ + -t ${{ parameters.TOPOLOGY }} \ + -o new_test_plan_id.txt \ + --min-worker ${{ parameters.MIN_WORKER }} \ + --max-worker ${{ parameters.MAX_WORKER }} \ + --lock-wait-timeout-seconds ${{ parameters.LOCK_WAIT_TIMEOUT_SECONDS }} \ + --test-set ${{ parameters.TEST_SET }} \ + --kvm-build-id $(KVM_BUILD_ID) \ + --kvm-image-branch "${{ parameters.KVM_IMAGE_BRANCH }}" \ + --deploy-mg-extra-params="${{ parameters.DEPLOY_MG_EXTRA_PARAMS }}" \ + --common-extra-params="${{ parameters.COMMON_EXTRA_PARAMS }}" \ + --vm-type ${{ parameters.VM_TYPE }} --num-asic ${{ parameters.NUM_ASIC }} \ + --image_url ${{ parameters.IMAGE_URL }} \ + --upgrade-image-param="${{ parameters.UPGRADE_IMAGE_PARAM }}" \ + --hwsku ${{ parameters.HWSKU }} \ + --test-plan-type ${{ parameters.TEST_PLAN_TYPE }} \ + --platform ${{ parameters.PLATFORM }} \ + --testbed-name "${{ parameters.TESTBED_NAME }}" \ + --scripts "${{ parameters.SCRIPTS }}" \ + --features "${{ parameters.FEATURES }}" \ + --scripts-exclude "${{ parameters.SCRIPTS_EXCLUDE }}" \ + --features-exclude "${{ parameters.FEATURES_EXCLUDE }}" \ + --specific-param='${{ parameters.SPECIFIC_PARAM }}' \ + --affinity='${{ parameters.AFFINITY }}' \ + --build-reason ${{ parameters.BUILD_REASON }} \ + --repo-name ${{ parameters.REPO_NAME }} \ + --mgmt-branch ${{ parameters.MGMT_BRANCH }} \ + --stop-on-failure ${{ parameters.STOP_ON_FAILURE }} \ + --retry-times ${{ parameters.RETRY_TIMES }} \ + --dump-kvm-if-fail ${{ parameters.DUMP_KVM_IF_FAIL }} \ + --requester "${{ parameters.REQUESTER }}" \ + --max-execute-seconds $((${{ parameters.MAX_RUN_TEST_MINUTES }} * 60)) \ + --test-plan-num ${{ parameters.TEST_PLAN_NUM }} + + TEST_PLAN_ID_LIST=( $(cat new_test_plan_id.txt) ) + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo "Created test plan $TEST_PLAN_ID" + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + done + TEST_PLAN_ID_LIST_STRING=$(printf "%s," "${TEST_PLAN_ID_LIST[@]}") + TEST_PLAN_ID_LIST_STRING=${TEST_PLAN_ID_LIST_STRING%,} + echo "##vso[task.setvariable variable=TEST_PLAN_ID_LIST_STRING]$TEST_PLAN_ID_LIST_STRING" displayName: "Trigger test" env: ELASTICTEST_MSAL_CLIENT_SECRET: $(ELASTICTEST_MSAL_CLIENT_SECRET) - - script: | - set -o - echo "Lock testbed" - - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - failure_count=0 - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED" - echo "[test_plan.py] poll LOCK_TESTBED status" - python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state LOCK_TESTBED - RET=$? - if [ $RET -ne 0 ]; then - ((failure_count++)) - fi - done - - if [ $failure_count -eq ${#TEST_PLAN_ID_LIST[@]} ]; then - echo "All testplan failed, cancel following steps" - exit 3 - fi + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -o + echo "Lock testbed" + + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + failure_count=0 + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + # When "LOCK_TESTBED" finish, it changes into "PREPARE_TESTBED" + echo "[test_plan.py] poll LOCK_TESTBED status" + python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state LOCK_TESTBED + RET=$? + if [ $RET -ne 0 ]; then + ((failure_count++)) + fi + done + + if [ $failure_count -eq ${#TEST_PLAN_ID_LIST[@]} ]; then + echo "All testplan failed, cancel following steps" + exit 3 + fi displayName: "Lock testbed" env: ELASTICTEST_MSAL_CLIENT_SECRET: $(ELASTICTEST_MSAL_CLIENT_SECRET) - - script: | - set -o - echo "Prepare testbed" - echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient" - - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - failure_count=0 - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - # When "PREPARE_TESTBED" finish, it changes into "EXECUTING" - echo "[test_plan.py] poll PREPARE_TESTBED status" - python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state PREPARE_TESTBED - RET=$? - if [ $RET -ne 0 ]; then - ((failure_count++)) - fi - done - - if [ "$failure_count" -eq ${#TEST_PLAN_ID_LIST[@]} ]; then - echo "All testplan failed, cancel following steps" - exit 3 - fi + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -o + echo "Prepare testbed" + echo "Preparing the testbed(add-topo, deploy-mg) may take 15-30 minutes. Before the testbed is ready, the progress of the test plan keeps displayed as 0, please be patient" + + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + failure_count=0 + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + # When "PREPARE_TESTBED" finish, it changes into "EXECUTING" + echo "[test_plan.py] poll PREPARE_TESTBED status" + python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state PREPARE_TESTBED + RET=$? + if [ $RET -ne 0 ]; then + ((failure_count++)) + fi + done + + if [ "$failure_count" -eq ${#TEST_PLAN_ID_LIST[@]} ]; then + echo "All testplan failed, cancel following steps" + exit 3 + fi displayName: "Prepare testbed" env: ELASTICTEST_MSAL_CLIENT_SECRET: $(ELASTICTEST_MSAL_CLIENT_SECRET) - - script: | - set -o - echo "Run test" - - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - failure_count=0 - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED" - echo "[test_plan.py] poll EXECUTING status" - python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state EXECUTING --expected-result ${{ parameters.EXPECTED_RESULT }} - RET=$? - if [ $RET -ne 0 ]; then - ((failure_count++)) - fi - done - - if [ $failure_count -eq ${#TEST_PLAN_ID_LIST[@]} ]; then - echo "All testplan failed, cancel following steps" - exit 3 - fi + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -o + echo "Run test" + + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + failure_count=0 + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + # When "EXECUTING" finish, it changes into "KVMDUMP", "FAILED", "CANCELLED" or "FINISHED" + echo "[test_plan.py] poll EXECUTING status" + python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state EXECUTING --expected-result ${{ parameters.EXPECTED_RESULT }} + RET=$? + if [ $RET -ne 0 ]; then + ((failure_count++)) + fi + done + + if [ $failure_count -eq ${#TEST_PLAN_ID_LIST[@]} ]; then + echo "All testplan failed, cancel following steps" + exit 3 + fi displayName: "Run test" timeoutInMinutes: ${{ parameters.MAX_RUN_TEST_MINUTES }} @@ -321,35 +346,45 @@ steps: ELASTICTEST_MSAL_CLIENT_SECRET: $(ELASTICTEST_MSAL_CLIENT_SECRET) - ${{ if eq(parameters.DUMP_KVM_IF_FAIL, 'True') }}: - - script: | - set -e - echo "KVM dump" - - echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - echo -e -n "\033[33mPlease visit Elastictest page \033[0m" - echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " - echo -e "\033[33mfor detailed test plan progress \033[0m" - # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED" - echo "##[group][test_plan.py] poll KVMDUMP status" - python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state KVMDUMP - done + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -e + echo "KVM dump" + + echo -e "\033[33mSONiC PR system-level test is powered by SONiC Elastictest, for any issue, please send email to sonicelastictest@microsoft.com \033[0m" + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + echo -e -n "\033[33mPlease visit Elastictest page \033[0m" + echo -n "$(FRONTEND_URL)/scheduler/testplan/$TEST_PLAN_ID " + echo -e "\033[33mfor detailed test plan progress \033[0m" + # When "KVMDUMP" finish, it changes into "FAILED", "CANCELLED" or "FINISHED" + echo "##[group][test_plan.py] poll KVMDUMP status" + python ./.azure-pipelines/test_plan.py poll -i $TEST_PLAN_ID --expected-state KVMDUMP + done condition: succeededOrFailed() displayName: "KVM dump" env: ELASTICTEST_MSAL_CLIENT_SECRET: $(ELASTICTEST_MSAL_CLIENT_SECRET) - - script: | - set -e - echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect." - IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" - for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" - do - python ./.azure-pipelines/test_plan.py cancel -i $TEST_PLAN_ID - done + - task: AzureCLI@2 + inputs: + azureSubscription: "SONiC-Automation" + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + set -e + echo "Try to cancel test plan $TEST_PLAN_ID, cancelling finished test plan has no effect." + IFS=',' read -ra TEST_PLAN_ID_LIST <<< "$TEST_PLAN_ID_LIST_STRING" + for TEST_PLAN_ID in "${TEST_PLAN_ID_LIST[@]}" + do + python ./.azure-pipelines/test_plan.py cancel -i $TEST_PLAN_ID + done condition: always() displayName: "Finalize running test plan" env: