From 56a7f53bdd4c9e4c7e9b3642579d4377a6619ac6 Mon Sep 17 00:00:00 2001 From: chunangli Date: Mon, 2 Sep 2024 14:15:40 +0800 Subject: [PATCH 1/5] debug test plan Signed-off-by: Chun'ang Li --- .azure-pipelines/run-test-elastictest-template.yml | 14 +++++++------- .azure-pipelines/test_plan.py | 4 +++- tests/common/plugins/sanity_check/__init__.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.azure-pipelines/run-test-elastictest-template.yml b/.azure-pipelines/run-test-elastictest-template.yml index 5b52a39a9fc..8cb35d20f0f 100644 --- a/.azure-pipelines/run-test-elastictest-template.yml +++ b/.azure-pipelines/run-test-elastictest-template.yml @@ -162,13 +162,13 @@ steps: curl -u :$(MSSONIC-TOKEN) "${{ parameters.MGMT_URL }}&commitOrBranch=${{ parameters.MGMT_BRANCH }}&api-version=5.0-preview.1&path=.azure-pipelines%2Fpr_test_scripts.yaml" -o ./.azure-pipelines/pr_test_scripts.yaml fi displayName: "Download pr script" - - ${{ else }}: - - ${{ if ne(parameters.MGMT_BRANCH, 'master') }}: - - script: | - # Else, sonic-mgmt repo, if not master branch, need to download test_plan.py - set -ex - curl "https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py" -o ./.azure-pipelines/test_plan.py - displayName: "Download test plan script" +# - ${{ else }}: +# - ${{ if ne(parameters.MGMT_BRANCH, 'master') }}: +# - script: | +# # Else, sonic-mgmt repo, if not master branch, need to download test_plan.py +# set -ex +# curl "https://raw.githubusercontent.com/sonic-net/sonic-mgmt/master/.azure-pipelines/test_plan.py" -o ./.azure-pipelines/test_plan.py +# displayName: "Download test plan script" - script: | # Check if azure cli is installed. If not, try to install it diff --git a/.azure-pipelines/test_plan.py b/.azure-pipelines/test_plan.py index c6308d4b210..0a743927f9e 100644 --- a/.azure-pipelines/test_plan.py +++ b/.azure-pipelines/test_plan.py @@ -380,7 +380,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte } start_time = time.time() http_exception_times = 0 - while (timeout < 0 or (time.time() - start_time) < timeout): + while timeout < 0 or (time.time() - start_time) < timeout: try: if self.with_auth: headers["Authorization"] = "Bearer {}".format(self.get_token()) @@ -409,6 +409,8 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte current_status = test_plan_status_factory(status) expected_status = test_plan_status_factory(expected_state) + print("current status: {}, expected status: {}".format(current_status, expected_status)) + if expected_status.get_status() == current_status.get_status(): current_status.print_logs(test_plan_id, resp_data, start_time) elif expected_status.get_status() < current_status.get_status(): diff --git a/tests/common/plugins/sanity_check/__init__.py b/tests/common/plugins/sanity_check/__init__.py index ca3da12ac99..d5cbbbb3c93 100644 --- a/tests/common/plugins/sanity_check/__init__.py +++ b/tests/common/plugins/sanity_check/__init__.py @@ -119,7 +119,7 @@ def do_checks(request, check_items, *args, **kwargs): return check_results -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def sanity_check_full(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo): logger.info("Prepare sanity check") From 5dd53fc7f6bbfa2d6ec176967e4f25961657f90f Mon Sep 17 00:00:00 2001 From: chunangli Date: Mon, 2 Sep 2024 15:55:47 +0800 Subject: [PATCH 2/5] debug test plan Signed-off-by: Chun'ang Li --- .azure-pipelines/test_plan.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/test_plan.py b/.azure-pipelines/test_plan.py index 0a743927f9e..ded67f08d60 100644 --- a/.azure-pipelines/test_plan.py +++ b/.azure-pipelines/test_plan.py @@ -74,7 +74,7 @@ def test_plan_status_factory(status): raise Exception("The status is not correct.") -class AbstractStatus(): +class AbstractStatus: def __init__(self, status): self.status = status @@ -409,7 +409,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte current_status = test_plan_status_factory(status) expected_status = test_plan_status_factory(expected_state) - print("current status: {}, expected status: {}".format(current_status, expected_status)) + print("current status: {}, expected status: {}".format(status, expected_state)) if expected_status.get_status() == current_status.get_status(): current_status.print_logs(test_plan_id, resp_data, start_time) @@ -432,7 +432,13 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte # We fail the step only if the step_status is "FAILED". # Other status such as "SKIPPED", "CANCELED" are considered successful. - if step_status == "FAILED": + """ + If step_status is None, means that current step was not executed but current status is in a post + status. For example: current is Failed, expect is Executing, and after prepare tb it went to fail, + so executing not started. In this scenario, step_status is None but current status is Failed. + So, return to failure. Otherwise, it will return to pass and cause inconsistency issues. + """ + if step_status == "FAILED" or (step_status is None and current_status is FailedStatus): # Print error type and message err_code = resp_data.get("runtime", {}).get("err_code", None) From c378c4c402ecc9c8d2b811eda7b9cdb451a659f7 Mon Sep 17 00:00:00 2001 From: chunangli Date: Mon, 2 Sep 2024 17:16:24 +0800 Subject: [PATCH 3/5] debug test plan Signed-off-by: Chun'ang Li --- .azure-pipelines/test_plan.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines/test_plan.py b/.azure-pipelines/test_plan.py index ded67f08d60..f61085f1f94 100644 --- a/.azure-pipelines/test_plan.py +++ b/.azure-pipelines/test_plan.py @@ -402,14 +402,14 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte if not resp_data: raise Exception("No valid data in response: {}".format(str(resp))) - status = resp_data.get("status", None) - result = resp_data.get("result", None) + current_tp_status = resp_data.get("status", None) + current_tp_result = resp_data.get("result", None) if expected_state: - current_status = test_plan_status_factory(status) + current_status = test_plan_status_factory(current_tp_status) expected_status = test_plan_status_factory(expected_state) - print("current status: {}, expected status: {}".format(status, expected_state)) + print("current status: {}, expected status: {}".format(current_tp_status, expected_state)) if expected_status.get_status() == current_status.get_status(): current_status.print_logs(test_plan_id, resp_data, start_time) @@ -438,7 +438,7 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte so executing not started. In this scenario, step_status is None but current status is Failed. So, return to failure. Otherwise, it will return to pass and cause inconsistency issues. """ - if step_status == "FAILED" or (step_status is None and current_status is FailedStatus): + if step_status == "FAILED" or (step_status is None and current_tp_status == "FAILED"): # Print error type and message err_code = resp_data.get("runtime", {}).get("err_code", None) @@ -451,23 +451,23 @@ def poll(self, test_plan_id, interval=60, timeout=-1, expected_state="", expecte raise Exception("Test plan id: {}, status: {}, result: {}, Elapsed {:.0f} seconds. " "Check {}/scheduler/testplan/{} for test plan status" - .format(test_plan_id, step_status, result, time.time() - start_time, + .format(test_plan_id, step_status, current_tp_result, time.time() - start_time, self.frontend_url, test_plan_id)) if expected_result: - if result != expected_result: + if current_tp_result != expected_result: raise Exception("Test plan id: {}, status: {}, result: {} not match expected result: {}, " "Elapsed {:.0f} seconds. " "Check {}/scheduler/testplan/{} for test plan status" - .format(test_plan_id, step_status, result, + .format(test_plan_id, step_status, current_tp_result, expected_result, time.time() - start_time, self.frontend_url, test_plan_id)) - print("Current status is {}".format(step_status)) + print("Current step status is {}".format(step_status)) return else: - print("Current state is {}, waiting for the state {}".format(status, expected_state)) + print("Current state is {}, waiting for the state {}".format(current_tp_status, expected_state)) time.sleep(interval) From 589423d5f46bd1dc61caaf3669abf9410f869e26 Mon Sep 17 00:00:00 2001 From: chunangli Date: Mon, 2 Sep 2024 17:17:16 +0800 Subject: [PATCH 4/5] debug test plan Signed-off-by: Chun'ang Li --- azure-pipelines.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 90917d89198..6dffedabd0d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,27 +23,27 @@ trigger: none name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) stages: -- stage: Pre_test - jobs: - - job: static_analysis - displayName: "Static Analysis" - timeoutInMinutes: 10 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/pre-commit-check.yml - - - job: validate_test_cases - displayName: "Validate Test Cases" - timeoutInMinutes: 20 - continueOnError: false - pool: sonic-common - steps: - - template: .azure-pipelines/pytest-collect-only.yml +#- stage: Pre_test +# jobs: +# - job: static_analysis +# displayName: "Static Analysis" +# timeoutInMinutes: 10 +# continueOnError: false +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/pre-commit-check.yml +# +# - job: validate_test_cases +# displayName: "Validate Test Cases" +# timeoutInMinutes: 20 +# continueOnError: false +# pool: sonic-common +# steps: +# - template: .azure-pipelines/pytest-collect-only.yml - stage: Test - dependsOn: Pre_test - condition: and(succeeded(), in(dependencies.Pre_test.result, 'Succeeded')) +# dependsOn: Pre_test +# condition: and(succeeded(), in(dependencies.Pre_test.result, 'Succeeded')) variables: - group: SONiC-Elastictest - name: inventory From 7355af61ed4b639e0cf829199173aa73bb244c6f Mon Sep 17 00:00:00 2001 From: chunangli Date: Tue, 3 Sep 2024 14:36:01 +0800 Subject: [PATCH 5/5] debug fix Signed-off-by: Chun'ang Li --- azure-pipelines.yml | 284 ++++++++++++++++++++++---------------------- 1 file changed, 142 insertions(+), 142 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6dffedabd0d..a17b1fef759 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -71,148 +71,148 @@ stages: KVM_IMAGE_BRANCH: $(BUILD_BRANCH) MGMT_BRANCH: $(BUILD_BRANCH) - - job: t0_2vlans_elastictest - displayName: "kvmtest-t0-2vlans by Elastictest" - timeoutInMinutes: 240 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t0 - TEST_SET: t0-2vlans - MIN_WORKER: $(T0_2VLANS_INSTANCE_NUM) - MAX_WORKER: $(T0_2VLANS_INSTANCE_NUM) - DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a" - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - - - job: t1_lag_elastictest - displayName: "kvmtest-t1-lag by Elastictest" - timeoutInMinutes: 240 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t1-lag - MIN_WORKER: $(T1_LAG_INSTANCE_NUM) - MAX_WORKER: $(T1_LAG_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - - - job: dualtor_elastictest - displayName: "kvmtest-dualtor-t0 by Elastictest" - timeoutInMinutes: 240 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: dualtor - MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM) - MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM) - COMMON_EXTRA_PARAMS: "--disable_loganalyzer " - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - - - job: multi_asic_elastictest - displayName: "kvmtest-multi-asic-t1-lag by Elastictest" - timeoutInMinutes: 240 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t1-8-lag - TEST_SET: multi-asic-t1-lag - MIN_WORKER: $(MULTI_ASIC_INSTANCE_NUM) - MAX_WORKER: $(MULTI_ASIC_INSTANCE_NUM) - NUM_ASIC: 4 - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - - - job: sonic_t0_elastictest - displayName: "kvmtest-t0-sonic by Elastictest" - timeoutInMinutes: 240 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t0-64-32 - MIN_WORKER: $(T0_SONIC_INSTANCE_NUM) - MAX_WORKER: $(T0_SONIC_INSTANCE_NUM) - TEST_SET: t0-sonic - COMMON_EXTRA_PARAMS: "--neighbor_type=sonic " - VM_TYPE: vsonic - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - - - job: dpu_elastictest - displayName: "kvmtest-dpu by Elastictest" - timeoutInMinutes: 240 - continueOnError: false - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: dpu - MIN_WORKER: $(T0_SONIC_INSTANCE_NUM) - MAX_WORKER: $(T0_SONIC_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - - - job: onboarding_elastictest_t0 - displayName: "onboarding t0 testcases by Elastictest - optional" - timeoutInMinutes: 240 - continueOnError: true - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t0 - STOP_ON_FAILURE: "False" - RETRY_TIMES: 0 - MIN_WORKER: $(T0_ONBOARDING_SONIC_INSTANCE_NUM) - MAX_WORKER: $(T0_ONBOARDING_SONIC_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - TEST_SET: onboarding_t0 - - - job: onboarding_elastictest_t1 - displayName: "onboarding t1 testcases by Elastictest - optional" - timeoutInMinutes: 240 - continueOnError: true - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: t1-lag - STOP_ON_FAILURE: "False" - RETRY_TIMES: 0 - MIN_WORKER: $(T1_LAG_ONBOARDING_INSTANCE_NUM) - MAX_WORKER: $(T1_LAG_ONBOARDING_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - TEST_SET: onboarding_t1 - - - job: onboarding_elastictest_dualtor - displayName: "onboarding dualtor testcases by Elastictest - optional" - timeoutInMinutes: 240 - continueOnError: true - pool: sonic-ubuntu-1c - steps: - - template: .azure-pipelines/run-test-elastictest-template.yml - parameters: - TOPOLOGY: dualtor - STOP_ON_FAILURE: "False" - RETRY_TIMES: 0 - MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM) - MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM) - KVM_IMAGE_BRANCH: $(BUILD_BRANCH) - MGMT_BRANCH: $(BUILD_BRANCH) - TEST_SET: onboarding_dualtor +# - job: t0_2vlans_elastictest +# displayName: "kvmtest-t0-2vlans by Elastictest" +# timeoutInMinutes: 240 +# continueOnError: false +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: t0 +# TEST_SET: t0-2vlans +# MIN_WORKER: $(T0_2VLANS_INSTANCE_NUM) +# MAX_WORKER: $(T0_2VLANS_INSTANCE_NUM) +# DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a" +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# +# - job: t1_lag_elastictest +# displayName: "kvmtest-t1-lag by Elastictest" +# timeoutInMinutes: 240 +# continueOnError: false +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: t1-lag +# MIN_WORKER: $(T1_LAG_INSTANCE_NUM) +# MAX_WORKER: $(T1_LAG_INSTANCE_NUM) +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# +# - job: dualtor_elastictest +# displayName: "kvmtest-dualtor-t0 by Elastictest" +# timeoutInMinutes: 240 +# continueOnError: false +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: dualtor +# MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM) +# MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM) +# COMMON_EXTRA_PARAMS: "--disable_loganalyzer " +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# +# - job: multi_asic_elastictest +# displayName: "kvmtest-multi-asic-t1-lag by Elastictest" +# timeoutInMinutes: 240 +# continueOnError: false +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: t1-8-lag +# TEST_SET: multi-asic-t1-lag +# MIN_WORKER: $(MULTI_ASIC_INSTANCE_NUM) +# MAX_WORKER: $(MULTI_ASIC_INSTANCE_NUM) +# NUM_ASIC: 4 +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# +# - job: sonic_t0_elastictest +# displayName: "kvmtest-t0-sonic by Elastictest" +# timeoutInMinutes: 240 +# continueOnError: false +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: t0-64-32 +# MIN_WORKER: $(T0_SONIC_INSTANCE_NUM) +# MAX_WORKER: $(T0_SONIC_INSTANCE_NUM) +# TEST_SET: t0-sonic +# COMMON_EXTRA_PARAMS: "--neighbor_type=sonic " +# VM_TYPE: vsonic +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# +# - job: dpu_elastictest +# displayName: "kvmtest-dpu by Elastictest" +# timeoutInMinutes: 240 +# continueOnError: false +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: dpu +# MIN_WORKER: $(T0_SONIC_INSTANCE_NUM) +# MAX_WORKER: $(T0_SONIC_INSTANCE_NUM) +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# +# - job: onboarding_elastictest_t0 +# displayName: "onboarding t0 testcases by Elastictest - optional" +# timeoutInMinutes: 240 +# continueOnError: true +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: t0 +# STOP_ON_FAILURE: "False" +# RETRY_TIMES: 0 +# MIN_WORKER: $(T0_ONBOARDING_SONIC_INSTANCE_NUM) +# MAX_WORKER: $(T0_ONBOARDING_SONIC_INSTANCE_NUM) +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# TEST_SET: onboarding_t0 +# +# - job: onboarding_elastictest_t1 +# displayName: "onboarding t1 testcases by Elastictest - optional" +# timeoutInMinutes: 240 +# continueOnError: true +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: t1-lag +# STOP_ON_FAILURE: "False" +# RETRY_TIMES: 0 +# MIN_WORKER: $(T1_LAG_ONBOARDING_INSTANCE_NUM) +# MAX_WORKER: $(T1_LAG_ONBOARDING_INSTANCE_NUM) +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# TEST_SET: onboarding_t1 +# +# - job: onboarding_elastictest_dualtor +# displayName: "onboarding dualtor testcases by Elastictest - optional" +# timeoutInMinutes: 240 +# continueOnError: true +# pool: sonic-ubuntu-1c +# steps: +# - template: .azure-pipelines/run-test-elastictest-template.yml +# parameters: +# TOPOLOGY: dualtor +# STOP_ON_FAILURE: "False" +# RETRY_TIMES: 0 +# MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM) +# MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM) +# KVM_IMAGE_BRANCH: $(BUILD_BRANCH) +# MGMT_BRANCH: $(BUILD_BRANCH) +# TEST_SET: onboarding_dualtor # - job: wan_elastictest # displayName: "kvmtest-wan by Elastictest"