Verify if the queue is blocked successfully in trimming case (sonic-net#21077)

In packet trimming test, it execute cmd sonic-db-cli CONFIG_DB hset 'QUEUE|Ethernet120|1' scheduler SCHEDULER_BLOCK_DATA_PLANE to block the queue of egress port to simulate buffer full and trigger trimmed packets. This cmd needs to be processed by swss.

In some trimming case, it executes ports shutdown/startup, config reload or reboot, which lead to BGP flap and lots of routes update message need to be processed by swss.

SWSS process all the messages one by one. If swss is busy to process route update messages, the block queue cmd may be processed few time later, which lead to the queue is not blocked immediately and case fail

Now add a check to make sure queue is blocked successfully before sending packets.

Signed-off-by: Guy Shemesh <gshemesh@nvidia.com>

Author: weiguo-nvidia

tests/packet_trimming/conftest.py

@@ -207,10 +207,6 @@ def setup_trimming(duthost, test_params):
         status = duthost.get_counter_poll_status()
         logger.info(f"Counter poll status: {status}")
 
-    with allure.step("Clear ports and queue counters"):
-        duthost.command("sonic-clear queuecounters")
-        duthost.command("sonic-clear counters")
-
     yield
 
     with allure.step("Disable trimming in buffer profile"):
@@ -303,3 +299,15 @@ def pytest_addoption(parser):
         required=False,
         help="reboot type such as reload, cold"
     )
+
+
+@pytest.fixture(scope="function", autouse=True)
+def clear_counters(duthost):
+    """
+    Clear all counters on the DUT.
+    """
+    duthost.shell("sonic-clear counters")
+    duthost.shell("sonic-clear queuecounters")
+    duthost.shell("sonic-clear switchcounters")
+
+    logger.info("Successfully cleared all counters on the DUT")

tests/packet_trimming/constants.py

@@ -74,6 +74,10 @@
 NORMAL_PACKET_DSCP = 4    # DSCP value for normal packet
 
 BLOCK_DATA_PLANE_SCHEDULER_NAME = "SCHEDULER_BLOCK_DATA_PLANE"
+SCHEDULER_TYPE = "DWRR"
+SCHEDULER_WEIGHT = 15
+SCHEDULER_PIR = 1
+
 DATA_PLANE_QUEUE_LIST = ["0", "1", "2", "3", "4", "5", "6"]
 DEFAULT_QUEUE_SCHEDULER_CONFIG = {
     "0": "scheduler.0",
@@ -165,7 +169,7 @@
 ]
 
 # Drop counter
-SWITCH_INTERVAL = 10000
+SWITCH_INTERVAL = 1000
 PORT_INTERVAL = 100
 QUEUE_INTERVAL = 100
 

tests/packet_trimming/packet_trimming_helper.py

@@ -23,7 +23,8 @@
                                              ACL_RULE_PRIORITY, ACL_TABLE_TYPE_NAME, ACL_RULE_NAME, SRV6_MY_SID_LIST,
                                              SRV6_INNER_SRC_IP, SRV6_INNER_DST_IP, DEFAULT_QUEUE_SCHEDULER_CONFIG,
                                              SRV6_UNIFORM_MODE, SRV6_OUTER_SRC_IPV6, SRV6_INNER_SRC_IPV6, ECN,
-                                             SRV6_INNER_DST_IPV6, SRV6_UN, ASYM_TC, ASYM_PORT_1_DSCP, ASYM_PORT_2_DSCP)
+                                             SRV6_INNER_DST_IPV6, SRV6_UN, ASYM_TC, ASYM_PORT_1_DSCP, ASYM_PORT_2_DSCP,
+                                             SCHEDULER_TYPE, SCHEDULER_WEIGHT, SCHEDULER_PIR)
 
 logger = logging.getLogger(__name__)
 
@@ -276,6 +277,101 @@ def generate_packet(duthost, packet_type, dst_addr, send_pkt_size, send_pkt_dscp
     return pkt, masked_exp_packet
 
 
+def get_scheduler_oid_by_attributes(duthost, **kwargs):
+    """
+    Find scheduler OID in ASIC_DB by matching its attributes.
+
+    Args:
+        duthost: DUT host object
+        **kwargs: Scheduler attributes to match
+            - type: Scheduler type (e.g., "DWRR", "STRICT")
+            - weight: Scheduling weight (e.g., 15)
+            - pir: Peak Information Rate (e.g., 1)
+
+    Returns:
+        str: OID of the matched scheduler, or None if not found
+    """
+    # Mapping from CONFIG_DB parameters to ASIC_DB SAI attributes
+    param_to_sai_attr = {
+        'type': 'SAI_SCHEDULER_ATTR_SCHEDULING_TYPE',
+        'weight': 'SAI_SCHEDULER_ATTR_SCHEDULING_WEIGHT',
+        'pir': 'SAI_SCHEDULER_ATTR_MAX_BANDWIDTH_RATE'
+    }
+
+    # Mapping for type values
+    type_value_mapping = {
+        'DWRR': 'SAI_SCHEDULING_TYPE_DWRR',
+        'STRICT': 'SAI_SCHEDULING_TYPE_STRICT'
+    }
+
+    # Build expected attributes dictionary
+    expected_attrs = {}
+    for param, value in kwargs.items():
+        if param not in param_to_sai_attr:
+            logger.warning(f"Unknown scheduler parameter: {param}")
+            continue
+
+        sai_attr = param_to_sai_attr[param]
+
+        # Convert type value to SAI format
+        if param == 'type':
+            if value in type_value_mapping:
+                expected_attrs[sai_attr] = type_value_mapping[value]
+            else:
+                logger.warning(f"Unknown scheduler type: {value}")
+                continue
+        else:
+            # For numeric values, convert to string for comparison
+            expected_attrs[sai_attr] = str(value)
+
+    logger.info(f"Looking for scheduler with attributes: {expected_attrs}")
+
+    # Get all scheduler OIDs from ASIC_DB
+    cmd_get_oids = 'redis-cli -n 1 keys "ASIC_STATE:SAI_OBJECT_TYPE_SCHEDULER:oid*"'
+    result = duthost.shell(cmd_get_oids)
+
+    if not result["stdout"].strip():
+        logger.warning("No schedulers found in ASIC_DB")
+        return None
+
+    oid_keys = result["stdout"].strip().split('\n')
+    logger.info(f"Found {len(oid_keys)} schedulers in ASIC_DB")
+
+    # Check each scheduler to find a match
+    for oid_key in oid_keys:
+        # Get all attributes of this scheduler
+        cmd_get_attrs = f'redis-cli -n 1 hgetall "{oid_key}"'
+        result = duthost.shell(cmd_get_attrs)
+
+        if not result["stdout"].strip():
+            continue
+
+        # Parse the attributes
+        lines = result["stdout"].strip().split('\n')
+        scheduler_attrs = {}
+        for i in range(0, len(lines), 2):
+            if i + 1 < len(lines):
+                scheduler_attrs[lines[i]] = lines[i + 1]
+
+        # Check if all expected attributes match
+        is_match = True
+        for attr_name, expected_value in expected_attrs.items():
+            actual_value = scheduler_attrs.get(attr_name)
+            if actual_value != expected_value:
+                is_match = False
+                break
+
+        if is_match:
+            # Extract the OID value (e.g., "0x160000000059aa")
+            oid_value = oid_key.split(':')[-1]
+            logger.info(f"Found matching scheduler OID: {oid_value}")
+            logger.debug(f"Scheduler attributes: {scheduler_attrs}")
+            return oid_value
+
+    logger.warning(f"No scheduler found matching attributes: {expected_attrs}")
+    return None
+
+
 def create_blocking_scheduler(duthost):
     """
     Create a blocking scheduler for limiting egress traffic
@@ -295,7 +391,7 @@ def create_blocking_scheduler(duthost):
         # Create blocking scheduler
         cmd_create = (
             f'sonic-db-cli CONFIG_DB hset "SCHEDULER|{BLOCK_DATA_PLANE_SCHEDULER_NAME}" '
-            f'"type" DWRR "weight" 15 "pir" 1'
+            f'"type" {SCHEDULER_TYPE} "weight" {SCHEDULER_WEIGHT} "pir" {SCHEDULER_PIR}'
         )
         duthost.shell(cmd_create)
         logger.info(f"Successfully created blocking scheduler: {BLOCK_DATA_PLANE_SCHEDULER_NAME}")
@@ -351,6 +447,42 @@ def validate_scheduler_configuration(duthost, dut_port, queue, expected_schedule
         return False
 
 
+def validate_scheduler_apply_to_queue_in_asic_db(duthost, scheduler_oid):
+    """
+    Validate that the scheduler is applied to queue in ASIC_DB.
+
+    Args:
+        duthost: DUT host object
+        scheduler_oid (str): Scheduler OID to validate (e.g., "0x160000000059aa")
+
+    Returns:
+        bool: True if applied to queue in ASIC_DB, False otherwise
+    """
+    logger.debug(f"Validating scheduler OID {scheduler_oid} in ASIC_DB")
+
+    # Dump ASIC_DB to a temporary file for faster searching
+    tmp_file = "/tmp/asic_db_scheduler_check.json"
+    dump_cmd = f"sonic-db-dump -n ASIC_DB -y > {tmp_file}"
+    duthost.shell(dump_cmd)
+
+    # Search for the scheduler OID in SAI_SCHEDULER_GROUP_ATTR_SCHEDULER_PROFILE_ID
+    cmd_grep_oid = f'grep "SAI_SCHEDULER_GROUP_ATTR_SCHEDULER_PROFILE_ID" {tmp_file} | grep -c "{scheduler_oid}"'
+    result = duthost.shell(cmd_grep_oid)
+
+    # Clean up temporary file
+    duthost.shell(f"rm -f {tmp_file}")
+
+    # Check if scheduler OID is found in ASIC_DB
+    count = int(result["stdout"].strip()) if result["stdout"].strip() else 0
+
+    if count > 0:
+        logger.debug(f"ASIC_DB scheduler validation successful: OID {scheduler_oid} found in {count} scheduler groups")
+        return True
+    else:
+        logger.debug(f"ASIC_DB scheduler validation failed: OID {scheduler_oid} not found in any scheduler group")
+        return False
+
+
 def disable_egress_data_plane(duthost, dut_port, queue):
     """
     Disable egress data plane for a specific queue on a specific port.
@@ -378,11 +510,20 @@ def disable_egress_data_plane(duthost, dut_port, queue):
     cmd_block_q = f"sonic-db-cli CONFIG_DB hset 'QUEUE|{dut_port}|{queue}' scheduler {BLOCK_DATA_PLANE_SCHEDULER_NAME}"
     duthost.shell(cmd_block_q)
 
-    # Wait for the blocking scheduler configuration to take effect
+    # Wait for the blocking scheduler configuration to take effect in CONFIG_DB
     pytest_assert(wait_until(60, 5, 0, validate_scheduler_configuration,
                              duthost, dut_port, queue, BLOCK_DATA_PLANE_SCHEDULER_NAME),
                   f"Blocking scheduler configuration failed for port {dut_port} queue {queue}")
 
+    # Get the blocking scheduler OID from ASIC_DB
+    scheduler_oid = get_scheduler_oid_by_attributes(duthost, type=SCHEDULER_TYPE,
+                                                    weight=SCHEDULER_WEIGHT, pir=SCHEDULER_PIR)
+    pytest_assert(scheduler_oid, "Failed to find blocking scheduler OID in ASIC_DB")
+
+    # Wait for the blocking scheduler configuration to take effect in ASIC_DB
+    pytest_assert(wait_until(60, 5, 0, validate_scheduler_apply_to_queue_in_asic_db, duthost, scheduler_oid),
+                  f"Scheduler OID {scheduler_oid} validation in ASIC_DB failed for port {dut_port} queue {queue}")
+
     logger.info(f"Successfully applied blocking scheduler to port {dut_port} queue {queue}")
 
     return original_scheduler