From b99f977c17797581217901bafb2eadef333f1b97 Mon Sep 17 00:00:00 2001 From: nnelluri-cisco Date: Thu, 12 Feb 2026 11:47:06 -0800 Subject: [PATCH 01/10] added new HA fixured to programme dash ha scope and ha set tables via proto utils Signed-off-by: nnelluri-cisco Signed-off-by: nnelluri --- .../common/ha/dash_ha_scope_config_table.json | 24 +- .../ha/dash_ha_set_dpu_config_table.json | 14 + tests/ha/conftest.py | 323 +++++++++++++++++- 3 files changed, 349 insertions(+), 12 deletions(-) create mode 100644 tests/common/ha/dash_ha_set_dpu_config_table.json diff --git a/tests/common/ha/dash_ha_scope_config_table.json b/tests/common/ha/dash_ha_scope_config_table.json index eee5aaf90f8..ad7ee1a870e 100644 --- a/tests/common/ha/dash_ha_scope_config_table.json +++ b/tests/common/ha/dash_ha_scope_config_table.json @@ -1,14 +1,20 @@ { - "DASH_HA_SET_CONFIG_TABLE": { - "ha-set-1001": { + "DASH_HA_SCOPE_CONFIG_TABLE": { + "vdpu0_0|haset0_0": { "version": "1.0", - "vip_v4": "192.168.100.1", - "vip_v6": "fd00::1001", - "vdpu_ids": ["dpu0", "dpu1"], - "scope": "dpu", - "pinned_vdpu_bfd_probe_states": ["up", "down"], - "preferred_vdpu_id": "dpu0", - "preferred_standalone_vdpu_index": 0 + "owner": "switch", + "disabled": "false", + "ha_set_id": "haset0_0", + "desired_ha_state": "active", + "approved_pending_operation_ids": "" + }, + "vdpu1_0|haset0_0": { + "version": "1.0", + "owner": "switch", + "disabled": "false", + "ha_set_id": "haset0_0", + "desired_ha_state": "standalone", + "approved_pending_operation_ids": "" } } } diff --git a/tests/common/ha/dash_ha_set_dpu_config_table.json b/tests/common/ha/dash_ha_set_dpu_config_table.json new file mode 100644 index 00000000000..869e92257c9 --- /dev/null +++ b/tests/common/ha/dash_ha_set_dpu_config_table.json @@ -0,0 +1,14 @@ +{ + "DASH_HA_SET_CONFIG_TABLE": { + "haset0_0": { + "version": "1.0", + "vip_v4": "3.2.1.0", + "vip_v6": "3:2::1:0", + "vdpu_ids": ["vdpu0_0", "vdpu1_0"], + "scope": "dpu", + "pinned_vdpu_bfd_probe_states": ["up", "down"], + "preferred_vdpu_id": "vdpu0_0", + "preferred_standalone_vdpu_index": "0" + } + } +} diff --git a/tests/ha/conftest.py b/tests/ha/conftest.py index a163b4038e8..3eb2743264d 100644 --- a/tests/ha/conftest.py +++ b/tests/ha/conftest.py @@ -1,11 +1,16 @@ +import logging import pytest -import time +from pathlib import Path +from collections import defaultdict +import re +import os import json +import ast +import time from tests.common.config_reload import config_reload -from pathlib import Path -from collections import defaultdict from tests.common.helpers.constants import DEFAULT_NAMESPACE +from tests.common.utilities import wait_until from common.ha.smartswitch_ha_helper import PtfTcpTestAdapter from common.ha.smartswitch_ha_io import SmartSwitchHaTrafficTest from common.ha.smartswitch_ha_helper import ( @@ -15,6 +20,8 @@ add_static_route_to_dut ) +logger = logging.getLogger(__name__) + @pytest.fixture(scope="module") def copy_files(ptfhost): @@ -408,3 +415,313 @@ def setup_ha_config(duthosts): final_cfg[f"DUT{switch_id}"] = cfg return final_cfg + + +def build_dash_ha_set_args(fields): + """ + Build args for DASH_HA_SET_CONFIG_TABLE + EXACTLY following the working CLI + """ + + version = str(fields["version"]) + if version.endswith(".0"): + version = version[:-2] + + return ( + f'version \\"{version}\\" ' + f'vip_v4 "{fields["vip_v4"]}" ' + f'vip_v6 "{fields["vip_v6"]}" ' + f'scope "{fields["scope"]}" ' + f'preferred_vdpu_id "{fields["preferred_vdpu_id"]}" ' + f'preferred_standalone_vdpu_index 0 ' + f'vdpu_ids \'["vdpu0_0","vdpu1_0"]\'' + ) + + +def build_dash_ha_scope_args(fields): + """ + Build args for DASH_HA_SCOPE_CONFIG_TABLE + EXACTLY following the working CLI + """ + + version = str(fields["version"]) + if version.endswith(".0"): + version = version[:-2] + + return ( + f'version \\"{version}\\" ' + f'disabled "{fields["disabled"]}" ' + f'desired_ha_state "{fields["desired_ha_state"]}" ' + f'ha_set_id "{fields["ha_set_id"]}" ' + f'owner "{fields["owner"]}"' + ) + + +def extract_pending_operations(text): + """ + Extract pending_operation_ids and pending_operation_types + and return list of (type, id) tuples. + """ + ids_match = re.search( + r'pending_operation_ids\s*\|\s*([^\|\r\n]+)', + text, + re.DOTALL, + ) + types_match = re.search( + r'pending_operation_types\s*\|\s*([^\|\r\n]+)', + text, + re.DOTALL, + ) + if not ids_match or not types_match: + return [] + + try: + ids = ast.literal_eval(f"'{ids_match.group(1)}'") + id_list = ids.split() + ids = id_list[0].split(',') + types = ast.literal_eval(f"'{types_match.group(1)}'") + type_list = types.split() + types = type_list[0].split(',') + except Exception: + return [] + + return list(zip(types, ids)) + + +def get_pending_operation_id(duthost, scope_key, expected_op_type): + """ + scope_key example: vdpu0_0:haset0_0 + expected_op_type example: ACTIVATE_ROLE + """ + cmd = ( + "docker exec dash-hadpu0 swbus-cli show hamgrd actor " + f"/hamgrd/0/ha-scope/{scope_key}" + ) + res = duthost.shell(cmd) + + pending_ops = extract_pending_operations(res["stdout"]) + + for op_type, op_id in pending_ops: + if op_type == expected_op_type: + return op_id + + return None + + +def build_dash_ha_scope_activate_args(fields, pending_id): + return ( + f'version \\"{fields["version"]}\\" ' + f'disabled {fields["disabled"]} ' + f'desired_ha_state "{fields["desired_ha_state"]}" ' + f'ha_set_id "{fields["ha_set_id"]}" ' + f'owner "{fields["owner"]}" ' + f'approved_pending_operation_ids ' + f'[\\\"{pending_id}\\\"]' + ) + + +def proto_utils_hset(duthost, table, key, args): + """ + Wrapper around proto_utils.py hset + + Args: + duthost: pytest duthost fixture + table (str): Redis table name + key (str): Redis key + args (str): Already-built proto args string + """ + cmd = ( + "docker exec swss python /etc/sonic/proto_utils.py hset " + f'"{table}:{key}" {args}' + ) + duthost.shell(cmd) + + +def wait_for_pending_operation_id( + duthost, + scope_key, + expected_op_type, + timeout=60, + interval=2, +): + """ + Wait until the expected pending_operation_id appears. + """ + pending_id = None + + def _condition(): + nonlocal pending_id + pending_id = get_pending_operation_id( + duthost, + scope_key, + expected_op_type, + ) + return pending_id is not None + + success = wait_until( + timeout, + interval, + 0, # REQUIRED delay argument + _condition, # condition callable + ) + + return pending_id if success else None + + +def extract_ha_state(text): + """ + Extract ha_state from swbus-cli output + """ + match = re.search(r'ha_state\s+\|\s+(\w+)', text) + return match.group(1) if match else None + + +def wait_for_ha_state( + duthost, + scope_key, + expected_state, + timeout=120, + interval=5, +): + """ + Wait until HA reaches the expected state + """ + def _check_ha_state(): + cmd = ( + "docker exec dash-hadpu0 swbus-cli show hamgrd actor " + f"/hamgrd/0/ha-scope/{scope_key}" + ) + res = duthost.shell(cmd) + return extract_ha_state(res["stdout"]) == expected_state + + success = wait_until( + timeout, + interval, + 0, + _check_ha_state + ) + + return success + + +@pytest.fixture(scope="module") +def setup_dash_ha_from_json(duthosts): + base_dir = "/data/tests/common/ha" + ha_set_file = os.path.join(base_dir, "dash_ha_set_dpu_config_table.json") + + with open(ha_set_file) as f: + ha_set_data = json.load(f)["DASH_HA_SET_CONFIG_TABLE"] + + # ------------------------------------------------- + # Step 1: Program HA SET on BOTH DUTs + # ------------------------------------------------- + for duthost in duthosts: + for key, fields in ha_set_data.items(): + proto_utils_hset( + duthost, + table="DASH_HA_SET_CONFIG_TABLE", + key=key, + args=build_dash_ha_set_args(fields), + ) + + # ------------------------------------------------- + # Step 2: Initial HA SCOPE per DUT + # ------------------------------------------------- + ha_scope_per_dut = [ + ( + "vdpu0_0:haset0_0", + { + "version": "1", + "disabled": "true", + "desired_ha_state": "active", + "ha_set_id": "haset0_0", + "owner": "dpu", + }, + ), + ( + "vdpu1_0:haset0_0", + { + "version": "1", + "disabled": "true", + "desired_ha_state": "unspecified", + "ha_set_id": "haset0_0", + "owner": "dpu", + }, + ), + ] + + for duthost, (key, fields) in zip(duthosts, ha_scope_per_dut): + proto_utils_hset( + duthost, + table="DASH_HA_SCOPE_CONFIG_TABLE", + key=key, + args=build_dash_ha_scope_args(fields), + ) + + +@pytest.fixture(scope="module") +def activate_dash_ha_from_json(duthosts): + # ------------------------------------------------- + # Step 4: Activate Role (using pending_operation_ids) + # ------------------------------------------------- + activate_scope_per_dut = [ + ( + "vdpu0_0:haset0_0", + { + "version": "1", + "disabled": "false", + "desired_ha_state": "active", + "ha_set_id": "haset0_0", + "owner": "dpu", + }, + ), + ( + "vdpu1_0:haset0_0", + { + "version": "1", + "disabled": "false", + "desired_ha_state": "unspecified", + "ha_set_id": "haset0_0", + "owner": "dpu", + }, + ), + ] + for duthost, (key, fields) in zip(duthosts, activate_scope_per_dut): + proto_utils_hset( + duthost, + table="DASH_HA_SCOPE_CONFIG_TABLE", + key=key, + args=build_dash_ha_scope_args(fields), + ) + + for duthost, (key, fields) in zip(duthosts, activate_scope_per_dut): + pending_id = wait_for_pending_operation_id( + duthost, + scope_key=key, + expected_op_type="activate_role", + timeout=60, + interval=2 + ) + assert pending_id, ( + f"Timed out waiting for active pending_operation_id " + f"for {duthost.hostname} scope {key}" + ) + + logger.info(f"DASH HA {duthost.hostname} found pending id {pending_id}") + proto_utils_hset( + duthost, + table="DASH_HA_SCOPE_CONFIG_TABLE", + key=key, + args=build_dash_ha_scope_activate_args(fields, pending_id), + ) + # Verify HA state using fields + assert wait_for_ha_state( + duthost, + scope_key=key, + expected_state="active", + timeout=120, + interval=5, + ), f"HA did not reach expected state for {key} on {duthost.hostname}" + logger.info(f"DASH HA Step-4 Activate Role completed for {duthost.hostname}") + logger.info("DASH HA Step-4 Activate Role completed") + yield From aa70da670d6c128c19ebee5fd8f9013848acf706 Mon Sep 17 00:00:00 2001 From: nnelluri-cisco Date: Thu, 12 Feb 2026 22:26:38 -0800 Subject: [PATCH 02/10] moved helper functions to ha_utils from conftest file Signed-off-by: nnelluri-cisco Signed-off-by: nnelluri --- tests/ha/conftest.py | 210 +++--------------------------- tests/ha/ha_utils.py | 302 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 195 deletions(-) create mode 100644 tests/ha/ha_utils.py diff --git a/tests/ha/conftest.py b/tests/ha/conftest.py index 3eb2743264d..0862a48f0a6 100644 --- a/tests/ha/conftest.py +++ b/tests/ha/conftest.py @@ -2,24 +2,31 @@ import pytest from pathlib import Path from collections import defaultdict -import re import os import json -import ast import time from tests.common.config_reload import config_reload from tests.common.helpers.constants import DEFAULT_NAMESPACE -from tests.common.utilities import wait_until -from common.ha.smartswitch_ha_helper import PtfTcpTestAdapter -from common.ha.smartswitch_ha_io import SmartSwitchHaTrafficTest -from common.ha.smartswitch_ha_helper import ( +from tests.common.ha.smartswitch_ha_helper import PtfTcpTestAdapter +from tests.common.ha.smartswitch_ha_io import SmartSwitchHaTrafficTest +from tests.common.ha.smartswitch_ha_helper import ( add_port_to_namespace, remove_namespace, add_static_route_to_ptf, add_static_route_to_dut ) +from ha_utils import ( + + build_dash_ha_scope_args, + wait_for_pending_operation_id, + build_dash_ha_scope_activate_args, + wait_for_ha_state, + build_dash_ha_set_args, + proto_utils_hset +) + logger = logging.getLogger(__name__) @@ -417,194 +424,7 @@ def setup_ha_config(duthosts): return final_cfg -def build_dash_ha_set_args(fields): - """ - Build args for DASH_HA_SET_CONFIG_TABLE - EXACTLY following the working CLI - """ - - version = str(fields["version"]) - if version.endswith(".0"): - version = version[:-2] - - return ( - f'version \\"{version}\\" ' - f'vip_v4 "{fields["vip_v4"]}" ' - f'vip_v6 "{fields["vip_v6"]}" ' - f'scope "{fields["scope"]}" ' - f'preferred_vdpu_id "{fields["preferred_vdpu_id"]}" ' - f'preferred_standalone_vdpu_index 0 ' - f'vdpu_ids \'["vdpu0_0","vdpu1_0"]\'' - ) - - -def build_dash_ha_scope_args(fields): - """ - Build args for DASH_HA_SCOPE_CONFIG_TABLE - EXACTLY following the working CLI - """ - - version = str(fields["version"]) - if version.endswith(".0"): - version = version[:-2] - - return ( - f'version \\"{version}\\" ' - f'disabled "{fields["disabled"]}" ' - f'desired_ha_state "{fields["desired_ha_state"]}" ' - f'ha_set_id "{fields["ha_set_id"]}" ' - f'owner "{fields["owner"]}"' - ) - - -def extract_pending_operations(text): - """ - Extract pending_operation_ids and pending_operation_types - and return list of (type, id) tuples. - """ - ids_match = re.search( - r'pending_operation_ids\s*\|\s*([^\|\r\n]+)', - text, - re.DOTALL, - ) - types_match = re.search( - r'pending_operation_types\s*\|\s*([^\|\r\n]+)', - text, - re.DOTALL, - ) - if not ids_match or not types_match: - return [] - - try: - ids = ast.literal_eval(f"'{ids_match.group(1)}'") - id_list = ids.split() - ids = id_list[0].split(',') - types = ast.literal_eval(f"'{types_match.group(1)}'") - type_list = types.split() - types = type_list[0].split(',') - except Exception: - return [] - - return list(zip(types, ids)) - - -def get_pending_operation_id(duthost, scope_key, expected_op_type): - """ - scope_key example: vdpu0_0:haset0_0 - expected_op_type example: ACTIVATE_ROLE - """ - cmd = ( - "docker exec dash-hadpu0 swbus-cli show hamgrd actor " - f"/hamgrd/0/ha-scope/{scope_key}" - ) - res = duthost.shell(cmd) - - pending_ops = extract_pending_operations(res["stdout"]) - - for op_type, op_id in pending_ops: - if op_type == expected_op_type: - return op_id - - return None - - -def build_dash_ha_scope_activate_args(fields, pending_id): - return ( - f'version \\"{fields["version"]}\\" ' - f'disabled {fields["disabled"]} ' - f'desired_ha_state "{fields["desired_ha_state"]}" ' - f'ha_set_id "{fields["ha_set_id"]}" ' - f'owner "{fields["owner"]}" ' - f'approved_pending_operation_ids ' - f'[\\\"{pending_id}\\\"]' - ) - - -def proto_utils_hset(duthost, table, key, args): - """ - Wrapper around proto_utils.py hset - - Args: - duthost: pytest duthost fixture - table (str): Redis table name - key (str): Redis key - args (str): Already-built proto args string - """ - cmd = ( - "docker exec swss python /etc/sonic/proto_utils.py hset " - f'"{table}:{key}" {args}' - ) - duthost.shell(cmd) - - -def wait_for_pending_operation_id( - duthost, - scope_key, - expected_op_type, - timeout=60, - interval=2, -): - """ - Wait until the expected pending_operation_id appears. - """ - pending_id = None - - def _condition(): - nonlocal pending_id - pending_id = get_pending_operation_id( - duthost, - scope_key, - expected_op_type, - ) - return pending_id is not None - - success = wait_until( - timeout, - interval, - 0, # REQUIRED delay argument - _condition, # condition callable - ) - - return pending_id if success else None - - -def extract_ha_state(text): - """ - Extract ha_state from swbus-cli output - """ - match = re.search(r'ha_state\s+\|\s+(\w+)', text) - return match.group(1) if match else None - - -def wait_for_ha_state( - duthost, - scope_key, - expected_state, - timeout=120, - interval=5, -): - """ - Wait until HA reaches the expected state - """ - def _check_ha_state(): - cmd = ( - "docker exec dash-hadpu0 swbus-cli show hamgrd actor " - f"/hamgrd/0/ha-scope/{scope_key}" - ) - res = duthost.shell(cmd) - return extract_ha_state(res["stdout"]) == expected_state - - success = wait_until( - timeout, - interval, - 0, - _check_ha_state - ) - - return success - - -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") def setup_dash_ha_from_json(duthosts): base_dir = "/data/tests/common/ha" ha_set_file = os.path.join(base_dir, "dash_ha_set_dpu_config_table.json") @@ -659,7 +479,7 @@ def setup_dash_ha_from_json(duthosts): ) -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") def activate_dash_ha_from_json(duthosts): # ------------------------------------------------- # Step 4: Activate Role (using pending_operation_ids) diff --git a/tests/ha/ha_utils.py b/tests/ha/ha_utils.py new file mode 100644 index 00000000000..e312b0ea9eb --- /dev/null +++ b/tests/ha/ha_utils.py @@ -0,0 +1,302 @@ +import logging +import re +import ast + +from tests.common.utilities import wait_until + +logger = logging.getLogger(__name__) + + +def build_dash_ha_scope_args(fields): + """ + Build args for DASH_HA_SCOPE_CONFIG_TABLE + EXACTLY following the working CLI + """ + + version = str(fields["version"]) + if version.endswith(".0"): + version = version[:-2] + + return ( + f'version \\"{version}\\" ' + f'disabled "{fields["disabled"]}" ' + f'desired_ha_state "{fields["desired_ha_state"]}" ' + f'ha_set_id "{fields["ha_set_id"]}" ' + f'owner "{fields["owner"]}"' + ) + + +def proto_utils_hset(duthost, table, key, args): + """ + Wrapper around proto_utils.py hset + + Args: + duthost: pytest duthost fixture + table (str): Redis table name + key (str): Redis key + args (str): Already-built proto args string + """ + cmd = ( + "docker exec swss python /etc/sonic/proto_utils.py hset " + f'"{table}:{key}" {args}' + ) + logger.debug(f"{duthost.hostname} running command: {cmd}") + out = duthost.shell(cmd) + logger.debug(f"{duthost.hostname} command output: {out}") + + +def build_dash_ha_set_args(fields): + """ + Build args for DASH_HA_SET_CONFIG_TABLE + EXACTLY following the working CLI + """ + + version = str(fields["version"]) + if version.endswith(".0"): + version = version[:-2] + + return ( + f'version \\"{version}\\" ' + f'vip_v4 "{fields["vip_v4"]}" ' + f'vip_v6 "{fields["vip_v6"]}" ' + f'scope "{fields["scope"]}" ' + f'preferred_vdpu_id "{fields["preferred_vdpu_id"]}" ' + f'preferred_standalone_vdpu_index 0 ' + f'vdpu_ids \'["vdpu0_0","vdpu1_0"]\'' + ) + + +def extract_pending_operations(text): + """ + Extract pending_operation_ids and pending_operation_types + and return list of (type, id) tuples. + """ + ids_match = re.search( + r'pending_operation_ids\s*\|\s*([^\|\r\n]+)', + text, + re.DOTALL, + ) + types_match = re.search( + r'pending_operation_types\s*\|\s*([^\|\r\n]+)', + text, + re.DOTALL, + ) + if not ids_match or not types_match: + return [] + + try: + ids = ast.literal_eval(f"'{ids_match.group(1)}'") + id_list = ids.split() + ids = id_list[0].split(',') + types = ast.literal_eval(f"'{types_match.group(1)}'") + type_list = types.split() + types = type_list[0].split(',') + except Exception: + return [] + + return list(zip(types, ids)) + + +def get_pending_operation_id(duthost, scope_key, timeout=60): + """ + scope_key example: vdpu0_0:haset0_0 + expected_op_type example: ACTIVATE_ROLE + """ + cmd = ( + "docker exec dash-hadpu0 swbus-cli show hamgrd actor " + f"/hamgrd/0/ha-scope/{scope_key}" + ) + + """ + Wait until the expected pending_operation_id appears. + """ + pending_id = None + expected_op_type = "activate_role" + + def _condition(): + nonlocal pending_id + res = duthost.shell(cmd) + pending_ops = extract_pending_operations(res["stdout"]) + + for op_type, op_id in pending_ops: + if op_type == expected_op_type: + pending_id = op_id + break + return pending_id is not None + + interval = 2 + success = wait_until( + timeout, + interval, + 0, # REQUIRED delay argument + _condition, # condition callable + ) + if not success: + logger.warning(f"{duthost.hostname} Timeout waiting for pending operation ID for scope {scope_key}") + return pending_id if success else None + + +def build_dash_ha_scope_activate_args(fields, pending_id): + return ( + f'version \\"{fields["version"]}\\" ' + f'disabled {fields["disabled"]} ' + f'desired_ha_state "{fields["desired_ha_state"]}" ' + f'ha_set_id "{fields["ha_set_id"]}" ' + f'owner "{fields["owner"]}" ' + f'approved_pending_operation_ids ' + f'[\\\"{pending_id}\\\"]' + ) + + +def verify_ha_state( + duthost, + scope_key, + expected_state, + timeout=120, + interval=5, +): + """ + Wait until HA reaches the expected state + """ + def _check_ha_state(): + cmd = ( + "docker exec dash-hadpu0 swbus-cli show hamgrd actor " + f"/hamgrd/0/ha-scope/{scope_key}" + ) + res = duthost.shell(cmd) + match = re.search(r'ha_state\s+\|\s+(\w+)', res["stdout"]) + return match.group(1) == expected_state + + success = wait_until(timeout, interval, 0, _check_ha_state) + + return success + + +def activate_primary_dash_ha(duthost, scope_key): + """ + Activate Role using pending_operation_ids + """ + fields = { + "version": "1", + "disabled": "false", + "desired_ha_state": "active", + "ha_set_id": "haset0_0", + "owner": "dpu", + } + return activate_dash_ha(duthost, scope_key, fields) + + +def activate_secondary_dash_ha(duthost, scope_key): + """ + Activate Role using pending_operation_ids + """ + fields = { + "version": "1", + "disabled": "false", + "desired_ha_state": "unspecified", + "ha_set_id": "haset0_0", + "owner": "dpu", + } + return activate_dash_ha(duthost, scope_key, fields) + + +def activate_dash_ha(duthost, scope_key, fields): + + proto_utils_hset( + duthost, + table="DASH_HA_SCOPE_CONFIG_TABLE", + key=scope_key, + args=build_dash_ha_scope_args(fields), + ) + + pending_id = get_pending_operation_id(duthost, scope_key, timeout=60) + assert pending_id, ( + f"Timed out waiting for active pending_operation_id " + f"for scope {scope_key}" + ) + proto_utils_hset( + duthost, + table="DASH_HA_SCOPE_CONFIG_TABLE", + key=scope_key, + args=build_dash_ha_scope_activate_args(fields, pending_id), + ) + + if verify_ha_state( + duthost, + scope_key, + expected_state="active", + timeout=120, + interval=5, + ): + logger.info(f"HA reached ACTIVE state for {scope_key}") + return True + else: + logger.warning(f"HA did not reach ACTIVE state for {scope_key}") + return False + + +def wait_for_pending_operation_id( + duthost, + scope_key, + expected_op_type, + timeout=60, + interval=2, +): + """ + Wait until the expected pending_operation_id appears. + """ + pending_id = None + + def _condition(): + nonlocal pending_id + pending_id = get_pending_operation_id( + duthost, + scope_key, + expected_op_type, + ) + return pending_id is not None + + success = wait_until( + timeout, + interval, + 0, # REQUIRED delay argument + _condition, # condition callable + ) + + return pending_id if success else None + + +def extract_ha_state(text): + """ + Extract ha_state from swbus-cli output + """ + match = re.search(r'ha_state\s+\|\s+(\w+)', text) + return match.group(1) if match else None + + +def wait_for_ha_state( + duthost, + scope_key, + expected_state, + timeout=120, + interval=5, +): + """ + Wait until HA reaches the expected state + """ + def _check_ha_state(): + cmd = ( + "docker exec dash-hadpu0 swbus-cli show hamgrd actor " + f"/hamgrd/0/ha-scope/{scope_key}" + ) + res = duthost.shell(cmd) + return extract_ha_state(res["stdout"]) == expected_state + + success = wait_until( + timeout, + interval, + 0, + _check_ha_state + ) + + return success From ae605303727505ff462f87e0306c016def7c575e Mon Sep 17 00:00:00 2001 From: nnelluri-cisco Date: Fri, 13 Feb 2026 09:22:26 -0800 Subject: [PATCH 03/10] added module scope for setup_dash_ha_from_json Signed-off-by: nnelluri-cisco Signed-off-by: nnelluri --- tests/ha/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ha/conftest.py b/tests/ha/conftest.py index 0862a48f0a6..c703faa03d9 100644 --- a/tests/ha/conftest.py +++ b/tests/ha/conftest.py @@ -424,7 +424,7 @@ def setup_ha_config(duthosts): return final_cfg -@pytest.fixture(scope="function") +@pytest.fixture(scope="module") def setup_dash_ha_from_json(duthosts): base_dir = "/data/tests/common/ha" ha_set_file = os.path.join(base_dir, "dash_ha_set_dpu_config_table.json") From e3fb952edeb78b99735caf31f8982d652c84d937 Mon Sep 17 00:00:00 2001 From: nnelluri-cisco Date: Sun, 15 Feb 2026 09:06:50 -0800 Subject: [PATCH 04/10] fixed re parsing code and corrected expected state for each DUT Signed-off-by: nnelluri-cisco Signed-off-by: nnelluri --- tests/ha/conftest.py | 5 +++-- tests/ha/ha_utils.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/ha/conftest.py b/tests/ha/conftest.py index c703faa03d9..5e7bfea8708 100644 --- a/tests/ha/conftest.py +++ b/tests/ha/conftest.py @@ -514,7 +514,7 @@ def activate_dash_ha_from_json(duthosts): args=build_dash_ha_scope_args(fields), ) - for duthost, (key, fields) in zip(duthosts, activate_scope_per_dut): + for idx, (duthost, (key, fields)) in enumerate(zip(duthosts, activate_scope_per_dut)): pending_id = wait_for_pending_operation_id( duthost, scope_key=key, @@ -535,10 +535,11 @@ def activate_dash_ha_from_json(duthosts): args=build_dash_ha_scope_activate_args(fields, pending_id), ) # Verify HA state using fields + expected_state = "active" if idx == 0 else "standby" assert wait_for_ha_state( duthost, scope_key=key, - expected_state="active", + expected_state=expected_state, timeout=120, interval=5, ), f"HA did not reach expected state for {key} on {duthost.hostname}" diff --git a/tests/ha/ha_utils.py b/tests/ha/ha_utils.py index e312b0ea9eb..ebf469c5f2f 100644 --- a/tests/ha/ha_utils.py +++ b/tests/ha/ha_utils.py @@ -270,7 +270,8 @@ def extract_ha_state(text): """ Extract ha_state from swbus-cli output """ - match = re.search(r'ha_state\s+\|\s+(\w+)', text) + text_str = str(text) + match = re.search(r'"ha_role":\s*"(\w+)"', text_str) return match.group(1) if match else None From abe804eea76a1b748f889af6e178fefafdafd7d9 Mon Sep 17 00:00:00 2001 From: nnelluri-cisco Date: Sun, 15 Feb 2026 13:57:01 -0800 Subject: [PATCH 05/10] Fix DASH HA proto encoding and activation flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Convert disabled boolean to lowercase string without quotes (False → "false") - Fix field type handling: strings quoted, booleans unquoted - Implement proper two-phase setup: disabled=true → false - Add pending_operation_id activation workflow - Fix build_dash_ha_scope_args() and build_dash_ha_scope_activate_args() Signed-off-by: nnelluri-cisco Signed-off-by: nnelluri --- tests/ha/conftest.py | 11 +++++------ tests/ha/ha_utils.py | 9 +++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/ha/conftest.py b/tests/ha/conftest.py index 5e7bfea8708..83c580d992d 100644 --- a/tests/ha/conftest.py +++ b/tests/ha/conftest.py @@ -479,7 +479,7 @@ def setup_dash_ha_from_json(duthosts): ) -@pytest.fixture(scope="function") +@pytest.fixture(scope="module") def activate_dash_ha_from_json(duthosts): # ------------------------------------------------- # Step 4: Activate Role (using pending_operation_ids) @@ -489,7 +489,7 @@ def activate_dash_ha_from_json(duthosts): "vdpu0_0:haset0_0", { "version": "1", - "disabled": "false", + "disabled": False, "desired_ha_state": "active", "ha_set_id": "haset0_0", "owner": "dpu", @@ -499,7 +499,7 @@ def activate_dash_ha_from_json(duthosts): "vdpu1_0:haset0_0", { "version": "1", - "disabled": "false", + "disabled": False, "desired_ha_state": "unspecified", "ha_set_id": "haset0_0", "owner": "dpu", @@ -513,13 +513,12 @@ def activate_dash_ha_from_json(duthosts): key=key, args=build_dash_ha_scope_args(fields), ) - for idx, (duthost, (key, fields)) in enumerate(zip(duthosts, activate_scope_per_dut)): pending_id = wait_for_pending_operation_id( duthost, scope_key=key, expected_op_type="activate_role", - timeout=60, + timeout=120, interval=2 ) assert pending_id, ( @@ -542,7 +541,7 @@ def activate_dash_ha_from_json(duthosts): expected_state=expected_state, timeout=120, interval=5, - ), f"HA did not reach expected state for {key} on {duthost.hostname}" + ), f"HA did not reach expected state {expected_state} for {key} on {duthost.hostname}" logger.info(f"DASH HA Step-4 Activate Role completed for {duthost.hostname}") logger.info("DASH HA Step-4 Activate Role completed") yield diff --git a/tests/ha/ha_utils.py b/tests/ha/ha_utils.py index ebf469c5f2f..57401906ee5 100644 --- a/tests/ha/ha_utils.py +++ b/tests/ha/ha_utils.py @@ -16,10 +16,11 @@ def build_dash_ha_scope_args(fields): version = str(fields["version"]) if version.endswith(".0"): version = version[:-2] + disabled_val = str(fields["disabled"]).lower() return ( f'version \\"{version}\\" ' - f'disabled "{fields["disabled"]}" ' + f'disabled {disabled_val} ' f'desired_ha_state "{fields["desired_ha_state"]}" ' f'ha_set_id "{fields["ha_set_id"]}" ' f'owner "{fields["owner"]}"' @@ -97,7 +98,7 @@ def extract_pending_operations(text): return list(zip(types, ids)) -def get_pending_operation_id(duthost, scope_key, timeout=60): +def get_pending_operation_id(duthost, scope_key, expected_op_type, timeout=60): """ scope_key example: vdpu0_0:haset0_0 expected_op_type example: ACTIVATE_ROLE @@ -111,7 +112,6 @@ def get_pending_operation_id(duthost, scope_key, timeout=60): Wait until the expected pending_operation_id appears. """ pending_id = None - expected_op_type = "activate_role" def _condition(): nonlocal pending_id @@ -137,9 +137,10 @@ def _condition(): def build_dash_ha_scope_activate_args(fields, pending_id): + disabled_val = str(fields["disabled"]).lower() return ( f'version \\"{fields["version"]}\\" ' - f'disabled {fields["disabled"]} ' + f'disabled {disabled_val} ' f'desired_ha_state "{fields["desired_ha_state"]}" ' f'ha_set_id "{fields["ha_set_id"]}" ' f'owner "{fields["owner"]}" ' From 5d0a9e619305581f4c78f82383355d837ad2460d Mon Sep 17 00:00:00 2001 From: nnelluri-cisco Date: Wed, 18 Feb 2026 16:21:23 -0800 Subject: [PATCH 06/10] addressed review comments Signed-off-by: nnelluri-cisco Signed-off-by: nnelluri --- tests/ha/conftest.py | 5 +-- tests/ha/ha_utils.py | 83 +++++++++++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/tests/ha/conftest.py b/tests/ha/conftest.py index 83c580d992d..da62cfcd340 100644 --- a/tests/ha/conftest.py +++ b/tests/ha/conftest.py @@ -17,7 +17,7 @@ add_static_route_to_dut ) -from ha_utils import ( +from tests.ha.ha_utils import ( build_dash_ha_scope_args, wait_for_pending_operation_id, @@ -426,7 +426,8 @@ def setup_ha_config(duthosts): @pytest.fixture(scope="module") def setup_dash_ha_from_json(duthosts): - base_dir = "/data/tests/common/ha" + current_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.join(current_dir, "..", "common", "ha") ha_set_file = os.path.join(base_dir, "dash_ha_set_dpu_config_table.json") with open(ha_set_file) as f: diff --git a/tests/ha/ha_utils.py b/tests/ha/ha_utils.py index 57401906ee5..4591f158758 100644 --- a/tests/ha/ha_utils.py +++ b/tests/ha/ha_utils.py @@ -1,6 +1,7 @@ import logging import re import ast +import json from tests.common.utilities import wait_until @@ -55,15 +56,23 @@ def build_dash_ha_set_args(fields): version = str(fields["version"]) if version.endswith(".0"): version = version[:-2] - + vdpu_ids = fields.get("vdpu_ids", ["vdpu0_0", "vdpu1_0"]) + if isinstance(vdpu_ids, list): + vdpu_ids_str = json.dumps(vdpu_ids) + elif isinstance(vdpu_ids, str): + # If already a JSON string, use as-is + vdpu_ids_str = vdpu_ids + else: + raise TypeError(f"vdpu_ids must be list or string, got {type(vdpu_ids)}") + standalone_index = fields.get("preferred_standalone_vdpu_index", 0) return ( f'version \\"{version}\\" ' f'vip_v4 "{fields["vip_v4"]}" ' f'vip_v6 "{fields["vip_v6"]}" ' f'scope "{fields["scope"]}" ' f'preferred_vdpu_id "{fields["preferred_vdpu_id"]}" ' - f'preferred_standalone_vdpu_index 0 ' - f'vdpu_ids \'["vdpu0_0","vdpu1_0"]\'' + f'preferred_standalone_vdpu_index {standalone_index} ' + f'vdpu_ids \'{vdpu_ids_str}\' ' ) @@ -98,42 +107,52 @@ def extract_pending_operations(text): return list(zip(types, ids)) -def get_pending_operation_id(duthost, scope_key, expected_op_type, timeout=60): +def get_pending_operation_id(duthost, scope_key, expected_op_type): """ - scope_key example: vdpu0_0:haset0_0 - expected_op_type example: ACTIVATE_ROLE + Get pending operation ID from HA scope (single query, no retry). + + Args: + duthost: DUT host object + scope_key: HA scope key (e.g., "vdpu0_0:haset0_0") + expected_op_type: Expected operation type (e.g., "activate_role") + + Returns: + str: Pending operation ID if found, None otherwise """ cmd = ( "docker exec dash-hadpu0 swbus-cli show hamgrd actor " f"/hamgrd/0/ha-scope/{scope_key}" ) - """ - Wait until the expected pending_operation_id appears. - """ - pending_id = None - - def _condition(): - nonlocal pending_id + try: res = duthost.shell(cmd) + + if res.get("rc", 0) != 0: + logger.debug( + f"{duthost.hostname} Command failed for scope {scope_key}: " + f"{res.get('stderr', '')}" + ) + return None + pending_ops = extract_pending_operations(res["stdout"]) for op_type, op_id in pending_ops: if op_type == expected_op_type: - pending_id = op_id - break - return pending_id is not None + logger.debug( + f"{duthost.hostname} Found pending_operation_id {op_id} " + f"for scope {scope_key}" + ) + return op_id + + logger.debug( + f"{duthost.hostname} No {expected_op_type} operation found. " + f"Available: {pending_ops}" + ) + return None - interval = 2 - success = wait_until( - timeout, - interval, - 0, # REQUIRED delay argument - _condition, # condition callable - ) - if not success: - logger.warning(f"{duthost.hostname} Timeout waiting for pending operation ID for scope {scope_key}") - return pending_id if success else None + except Exception as e: + logger.debug(f"{duthost.hostname} Exception: {e}") + return None def build_dash_ha_scope_activate_args(fields, pending_id): @@ -173,7 +192,7 @@ def _check_ha_state(): return success -def activate_primary_dash_ha(duthost, scope_key): +def activate_primary_dash_ha(duthost, scope_key, expected_op_type): """ Activate Role using pending_operation_ids """ @@ -184,10 +203,10 @@ def activate_primary_dash_ha(duthost, scope_key): "ha_set_id": "haset0_0", "owner": "dpu", } - return activate_dash_ha(duthost, scope_key, fields) + return activate_dash_ha(duthost, scope_key, fields, expected_op_type) -def activate_secondary_dash_ha(duthost, scope_key): +def activate_secondary_dash_ha(duthost, scope_key, expected_op_type): """ Activate Role using pending_operation_ids """ @@ -198,10 +217,10 @@ def activate_secondary_dash_ha(duthost, scope_key): "ha_set_id": "haset0_0", "owner": "dpu", } - return activate_dash_ha(duthost, scope_key, fields) + return activate_dash_ha(duthost, scope_key, fields, expected_op_type) -def activate_dash_ha(duthost, scope_key, fields): +def activate_dash_ha(duthost, scope_key, fields, expected_op_type): proto_utils_hset( duthost, @@ -210,7 +229,7 @@ def activate_dash_ha(duthost, scope_key, fields): args=build_dash_ha_scope_args(fields), ) - pending_id = get_pending_operation_id(duthost, scope_key, timeout=60) + pending_id = get_pending_operation_id(duthost, scope_key, expected_op_type, timeout=60) assert pending_id, ( f"Timed out waiting for active pending_operation_id " f"for scope {scope_key}" From 9fc64a6008fd28d58e16b0adc5c81bf46cdd2add Mon Sep 17 00:00:00 2001 From: nnelluri-cisco Date: Thu, 19 Feb 2026 18:26:05 -0800 Subject: [PATCH 07/10] Fix activate_dash_ha to use wait_for_pending_operation_id with timeout get_pending_operation_id() was refactored to no longer accept a timeout parameter. Switch to wait_for_pending_operation_id() which supports the timeout argument to avoid a TypeError at runtime. Signed-off-by: nnelluri --- tests/ha/ha_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ha/ha_utils.py b/tests/ha/ha_utils.py index 4591f158758..4612dd7dde5 100644 --- a/tests/ha/ha_utils.py +++ b/tests/ha/ha_utils.py @@ -229,7 +229,7 @@ def activate_dash_ha(duthost, scope_key, fields, expected_op_type): args=build_dash_ha_scope_args(fields), ) - pending_id = get_pending_operation_id(duthost, scope_key, expected_op_type, timeout=60) + pending_id = wait_for_pending_operation_id(duthost, scope_key, expected_op_type, timeout=60) assert pending_id, ( f"Timed out waiting for active pending_operation_id " f"for scope {scope_key}" From 90b772c9607fe38119adf5fe3ef93748c22bad80 Mon Sep 17 00:00:00 2001 From: nnelluri Date: Thu, 19 Feb 2026 20:10:09 -0800 Subject: [PATCH 08/10] trigger pipeline Signed-off-by: nnelluri From 3d7c7d03ad3a250c50260f64378bb7300e6b10ec Mon Sep 17 00:00:00 2001 From: nnelluri Date: Mon, 23 Feb 2026 14:34:11 -0800 Subject: [PATCH 09/10] Fix key separator and type mismatch in DASH HA JSON config files - Change scope config keys from '|' to ':' separator to match fixture usage - Change preferred_standalone_vdpu_index from string "0" to int 0 Signed-off-by: nnelluri --- tests/common/ha/dash_ha_scope_config_table.json | 4 ++-- tests/common/ha/dash_ha_set_dpu_config_table.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/common/ha/dash_ha_scope_config_table.json b/tests/common/ha/dash_ha_scope_config_table.json index ad7ee1a870e..1754b603fa6 100644 --- a/tests/common/ha/dash_ha_scope_config_table.json +++ b/tests/common/ha/dash_ha_scope_config_table.json @@ -1,6 +1,6 @@ { "DASH_HA_SCOPE_CONFIG_TABLE": { - "vdpu0_0|haset0_0": { + "vdpu0_0:haset0_0": { "version": "1.0", "owner": "switch", "disabled": "false", @@ -8,7 +8,7 @@ "desired_ha_state": "active", "approved_pending_operation_ids": "" }, - "vdpu1_0|haset0_0": { + "vdpu1_0:haset0_0": { "version": "1.0", "owner": "switch", "disabled": "false", diff --git a/tests/common/ha/dash_ha_set_dpu_config_table.json b/tests/common/ha/dash_ha_set_dpu_config_table.json index 869e92257c9..a65d6b82df9 100644 --- a/tests/common/ha/dash_ha_set_dpu_config_table.json +++ b/tests/common/ha/dash_ha_set_dpu_config_table.json @@ -8,7 +8,7 @@ "scope": "dpu", "pinned_vdpu_bfd_probe_states": ["up", "down"], "preferred_vdpu_id": "vdpu0_0", - "preferred_standalone_vdpu_index": "0" + "preferred_standalone_vdpu_index": 0 } } } From cfb49eeffafa2978ea8afc029a0be2a4bff7eb41 Mon Sep 17 00:00:00 2001 From: nnelluri Date: Mon, 23 Feb 2026 22:16:54 -0800 Subject: [PATCH 10/10] trigger pipeline Signed-off-by: nnelluri