[Draft, not merge]Mock sanity failure

azure-pipelines.yml

@@ -39,123 +39,124 @@ stages:
     steps:
     - template: .azure-pipelines/pytest-collect-only.yml
 
-- stage: Test
-  dependsOn: Pre_test
-  condition: and(succeeded(), in(dependencies.Pre_test.result, 'Succeeded'))
-  variables:
-  - group: SONiC-Elastictest
-  - name: inventory
-    value: veos_vtb
-  - name: testbed_file
-    value: vtestbed.yaml
-  - group: GIT_SECRETS
 
-  jobs:
-  - job: t0_elastictest
-    displayName: "kvmtest-t0 by Elastictest"
-    timeoutInMinutes: 240
-    continueOnError: false
-    pool: ubuntu-20.04
-    steps:
-    - template: .azure-pipelines/run-test-elastictest-template.yml
-      parameters:
-        TOPOLOGY: t0
-        MIN_WORKER: $(T0_INSTANCE_NUM)
-        MAX_WORKER: $(T0_INSTANCE_NUM)
-        KVM_IMAGE_BRANCH: "master"
-        MGMT_BRANCH: "master"
-
-  - job: t0_2vlans_elastictest
-    displayName: "kvmtest-t0-2vlans by Elastictest"
-    timeoutInMinutes: 240
-    continueOnError: false
-    pool: ubuntu-20.04
-    steps:
-    - template: .azure-pipelines/run-test-elastictest-template.yml
-      parameters:
-        TOPOLOGY: t0
-        TEST_SET: t0-2vlans
-        MIN_WORKER: $(T0_2VLANS_INSTANCE_NUM)
-        MAX_WORKER: $(T0_2VLANS_INSTANCE_NUM)
-        DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a"
-        KVM_IMAGE_BRANCH: "master"
-        MGMT_BRANCH: "master"
-
-  - job: t1_lag_elastictest
-    displayName: "kvmtest-t1-lag by Elastictest"
-    timeoutInMinutes: 240
-    continueOnError: false
-    pool: ubuntu-20.04
-    steps:
-    - template: .azure-pipelines/run-test-elastictest-template.yml
-      parameters:
-        TOPOLOGY: t1-lag
-        MIN_WORKER: $(T1_LAG_INSTANCE_NUM)
-        MAX_WORKER: $(T1_LAG_INSTANCE_NUM)
-        KVM_IMAGE_BRANCH: "master"
-        MGMT_BRANCH: "master"
-
-  - job: dualtor_elastictest
-    displayName: "kvmtest-dualtor-t0 by Elastictest"
-    timeoutInMinutes: 240
-    continueOnError: false
-    pool: ubuntu-20.04
-    steps:
-      - template: .azure-pipelines/run-test-elastictest-template.yml
-        parameters:
-          TOPOLOGY: dualtor
-          MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
-          MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
-          COMMON_EXTRA_PARAMS: "--disable_loganalyzer "
-          KVM_IMAGE_BRANCH: "master"
-          MGMT_BRANCH: "master"
-
-  - job: multi_asic_elastictest
-    displayName: "kvmtest-multi-asic-t1-lag by Elastictest"
-    timeoutInMinutes: 240
-    continueOnError: false
-    pool: ubuntu-20.04
-    steps:
-      - template: .azure-pipelines/run-test-elastictest-template.yml
-        parameters:
-          TOPOLOGY: t1-8-lag
-          TEST_SET: multi-asic-t1-lag
-          MIN_WORKER: $(MULTI_ASIC_INSTANCE_NUM)
-          MAX_WORKER: $(MULTI_ASIC_INSTANCE_NUM)
-          NUM_ASIC: 4
-          KVM_IMAGE_BRANCH: "master"
-          MGMT_BRANCH: "master"
-
-  - job: sonic_t0_elastictest
-    displayName: "kvmtest-t0-sonic by Elastictest"
-    timeoutInMinutes: 240
-    continueOnError: false
-    pool: ubuntu-20.04
-    steps:
-      - template: .azure-pipelines/run-test-elastictest-template.yml
-        parameters:
-          TOPOLOGY: t0-64-32
-          MIN_WORKER: $(T0_SONIC_INSTANCE_NUM)
-          MAX_WORKER: $(T0_SONIC_INSTANCE_NUM)
-          TEST_SET: t0-sonic
-          COMMON_EXTRA_PARAMS: "--neighbor_type=sonic "
-          VM_TYPE: vsonic
-          KVM_IMAGE_BRANCH: "master"
-          MGMT_BRANCH: "master"
-
-  - job: dpu_elastictest
-    displayName: "kvmtest-dpu by Elastictest"
-    timeoutInMinutes: 240
-    continueOnError: false
-    pool: ubuntu-20.04
-    steps:
-      - template: .azure-pipelines/run-test-elastictest-template.yml
-        parameters:
-          TOPOLOGY: dpu
-          MIN_WORKER: $(T0_SONIC_INSTANCE_NUM)
-          MAX_WORKER: $(T0_SONIC_INSTANCE_NUM)
-          KVM_IMAGE_BRANCH: "master"
-          MGMT_BRANCH: "master"
+#- stage: Test
+#  dependsOn: Pre_test
+#  condition: and(succeeded(), in(dependencies.Pre_test.result, 'Succeeded'))
+#  variables:
+#  - group: SONiC-Elastictest
+#  - name: inventory
+#    value: veos_vtb
+#  - name: testbed_file
+#    value: vtestbed.yaml
+#  - group: GIT_SECRETS
+#
+#  jobs:
+#  - job: t0_elastictest
+#    displayName: "kvmtest-t0 by Elastictest"
+#    timeoutInMinutes: 240
+#    continueOnError: false
+#    pool: ubuntu-20.04
+#    steps:
+#    - template: .azure-pipelines/run-test-elastictest-template.yml
+#      parameters:
+#        TOPOLOGY: t0
+#        MIN_WORKER: $(T0_INSTANCE_NUM)
+#        MAX_WORKER: $(T0_INSTANCE_NUM)
+#        KVM_IMAGE_BRANCH: "master"
+#        MGMT_BRANCH: "master"
+#
+#  - job: t0_2vlans_elastictest
+#    displayName: "kvmtest-t0-2vlans by Elastictest"
+#    timeoutInMinutes: 240
+#    continueOnError: false
+#    pool: ubuntu-20.04
+#    steps:
+#    - template: .azure-pipelines/run-test-elastictest-template.yml
+#      parameters:
+#        TOPOLOGY: t0
+#        TEST_SET: t0-2vlans
+#        MIN_WORKER: $(T0_2VLANS_INSTANCE_NUM)
+#        MAX_WORKER: $(T0_2VLANS_INSTANCE_NUM)
+#        DEPLOY_MG_EXTRA_PARAMS: "-e vlan_config=two_vlan_a"
+#        KVM_IMAGE_BRANCH: "master"
+#        MGMT_BRANCH: "master"
+#
+#  - job: t1_lag_elastictest
+#    displayName: "kvmtest-t1-lag by Elastictest"
+#    timeoutInMinutes: 240
+#    continueOnError: false
+#    pool: ubuntu-20.04
+#    steps:
+#    - template: .azure-pipelines/run-test-elastictest-template.yml
+#      parameters:
+#        TOPOLOGY: t1-lag
+#        MIN_WORKER: $(T1_LAG_INSTANCE_NUM)
+#        MAX_WORKER: $(T1_LAG_INSTANCE_NUM)
+#        KVM_IMAGE_BRANCH: "master"
+#        MGMT_BRANCH: "master"
+#
+#  - job: dualtor_elastictest
+#    displayName: "kvmtest-dualtor-t0 by Elastictest"
+#    timeoutInMinutes: 240
+#    continueOnError: false
+#    pool: ubuntu-20.04
+#    steps:
+#      - template: .azure-pipelines/run-test-elastictest-template.yml
+#        parameters:
+#          TOPOLOGY: dualtor
+#          MIN_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
+#          MAX_WORKER: $(T0_DUALTOR_INSTANCE_NUM)
+#          COMMON_EXTRA_PARAMS: "--disable_loganalyzer "
+#          KVM_IMAGE_BRANCH: "master"
+#          MGMT_BRANCH: "master"
+#
+#  - job: multi_asic_elastictest
+#    displayName: "kvmtest-multi-asic-t1-lag by Elastictest"
+#    timeoutInMinutes: 240
+#    continueOnError: false
+#    pool: ubuntu-20.04
+#    steps:
+#      - template: .azure-pipelines/run-test-elastictest-template.yml
+#        parameters:
+#          TOPOLOGY: t1-8-lag
+#          TEST_SET: multi-asic-t1-lag
+#          MIN_WORKER: $(MULTI_ASIC_INSTANCE_NUM)
+#          MAX_WORKER: $(MULTI_ASIC_INSTANCE_NUM)
+#          NUM_ASIC: 4
+#          KVM_IMAGE_BRANCH: "master"
+#          MGMT_BRANCH: "master"
+#
+#  - job: sonic_t0_elastictest
+#    displayName: "kvmtest-t0-sonic by Elastictest"
+#    timeoutInMinutes: 240
+#    continueOnError: false
+#    pool: ubuntu-20.04
+#    steps:
+#      - template: .azure-pipelines/run-test-elastictest-template.yml
+#        parameters:
+#          TOPOLOGY: t0-64-32
+#          MIN_WORKER: $(T0_SONIC_INSTANCE_NUM)
+#          MAX_WORKER: $(T0_SONIC_INSTANCE_NUM)
+#          TEST_SET: t0-sonic
+#          COMMON_EXTRA_PARAMS: "--neighbor_type=sonic "
+#          VM_TYPE: vsonic
+#          KVM_IMAGE_BRANCH: "master"
+#          MGMT_BRANCH: "master"
+#
+#  - job: dpu_elastictest
+#    displayName: "kvmtest-dpu by Elastictest"
+#    timeoutInMinutes: 240
+#    continueOnError: false
+#    pool: ubuntu-20.04
+#    steps:
+#      - template: .azure-pipelines/run-test-elastictest-template.yml
+#        parameters:
+#          TOPOLOGY: dpu
+#          MIN_WORKER: $(T0_SONIC_INSTANCE_NUM)
+#          MAX_WORKER: $(T0_SONIC_INSTANCE_NUM)
+#          KVM_IMAGE_BRANCH: "master"
+#          MGMT_BRANCH: "master"
 
 #  - job: wan_elastictest
 #    displayName: "kvmtest-wan by Elastictest"

tests/common/plugins/sanity_check/__init__.py

@@ -2,6 +2,7 @@
 import logging
 import copy
 import json
+import random
 
 import pytest
 
@@ -216,6 +217,10 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo):
         # Workaround for pytest requirement.
         # Each possibly used check fixture must be executed in setup phase. Otherwise there could be teardown error.
         request.getfixturevalue(item)
+    logger.info("sanity node name is: {}".format(request.node.name))
+    if "pretest" in request.node.name or "posttest" in request.node.name:
+        logger.info("Skip sanity testKKK")
+        pre_check_items = []
 
     if pre_check_items:
         logger.info("Start pre-test sanity checks")
@@ -231,6 +236,14 @@ def sanity_check(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo):
                      json.dumps(check_results, indent=4, default=fallback_serializer))
 
         failed_results = [result for result in check_results if result['failed']]
+        num = random.randint(1, 9)
+        logger.info("num is: {}".format(num))
+
+        logger.info("request.node.name is: {}".format(request.node.name))
+        # if "test_bgp_fact" in request.node.name and num < 5:
+        #     failed_results = [1]
+        if num < 5:
+            failed_results = [1]
         if failed_results:
             if not allow_recover:
                 request.config.cache.set("pre_sanity_check_failed", True)

tests/common/plugins/sanity_check/recover.py

@@ -183,6 +183,9 @@ def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_result
 
 def recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, recover_method):
     logger.warning("Try to recover %s using method %s" % (dut.hostname, recover_method))
+    return
+    if recover_method == "config_reload" and config_force_option_supported(dut):
+        recover_method = "config_reload_f"
 
     method = constants.RECOVER_METHODS[recover_method]
     wait_time = method['recover_wait']