diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml index c53dd9dfcb3..de0d5f2a134 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions.yaml @@ -2466,6 +2466,12 @@ generic_config_updater: conditions: - "('t2' == topo_type) and (release in ['201811', '201911', '202012', '202205', '202211', '202305', '202311'])" +generic_config_updater/add_cluster/test_add_cluster.py: + skip: + reason: This test case either cannot pass or should be skipped on virtual chassis + conditions: + - asic_type in ['vs'] + generic_config_updater/test_bgp_prefix.py::test_bgp_prefix_tc1_suite: skip: reason: "Cisco 8122 backend compute ai platform is not supported. Skip on VS platform due to low success rate." diff --git a/tests/common/plugins/conditional_mark/tests_mark_conditions_vs_t2.yaml b/tests/common/plugins/conditional_mark/tests_mark_conditions_vs_t2.yaml index 1cf226224ad..6cd766df8db 100644 --- a/tests/common/plugins/conditional_mark/tests_mark_conditions_vs_t2.yaml +++ b/tests/common/plugins/conditional_mark/tests_mark_conditions_vs_t2.yaml @@ -255,6 +255,11 @@ everflow/test_everflow_testbed.py: - asic_type in ['vs'] and 't2' in topo_name - https://github.com/sonic-net/sonic-mgmt/issues/20257 reason: This test case takes too much time on virtual chassis +generic_config_updater/add_cluster/test_add_cluster.py: + skip: + conditions: + - asic_type in ['vs'] and 't2' in topo_name + reason: This test case either cannot pass or should be skipped on virtual chassis generic_config_updater/test_bgp_prefix.py: skip: conditions: diff --git a/tests/common/snappi_tests/common_helpers.py b/tests/common/snappi_tests/common_helpers.py index 33620f0882f..0f14da32f46 100644 --- a/tests/common/snappi_tests/common_helpers.py +++ b/tests/common/snappi_tests/common_helpers.py @@ -25,10 +25,10 @@ from ipaddress import IPv6Network, IPv6Address import ipaddress from random import getrandbits +from tests.common.helpers.assertions import pytest_assert from tests.common.portstat_utilities import parse_portstat from collections import defaultdict from tests.conftest import parse_override -from tests.common.helpers.assertions import pytest_assert from tests.common.utilities import wait_until logger = logging.getLogger(__name__) @@ -1322,12 +1322,13 @@ def start_pfcwd_fwd(duthost, asic_value=None): format(asic_value)) -def clear_counters(duthost, port): +def clear_counters(duthost, port=None, namespace=None): """ Clear PFC, Queuecounters, Drop and generic counters from SONiC CLI. Args: duthost (Ansible host instance): Device under test port (str): port name + namespace (str): namespace name in case of multi asic duthost Returns: None """ @@ -1343,8 +1344,15 @@ def clear_counters(duthost, port): duthost.command("sonic-clear queue watermark all \n") if (duthost.is_multi_asic): - asic = duthost.get_port_asic_instance(port).get_asic_namespace() - duthost.command("sudo ip netns exec {} sonic-clear dropcounters \n".format(asic)) + pytest_assert( + port or namespace, + 'Cannot clear counters in case of multi asic, either port or namespace needs to be provided.' + ) + if not namespace: + namespace = duthost.get_port_asic_instance(port).get_asic_namespace() + duthost.command("sudo ip netns exec {} sonic-clear dropcounters \n".format(namespace)) + else: + duthost.command("sonic-clear dropcounters \n") def get_interface_stats(duthost, port): diff --git a/tests/generic_config_updater/add_cluster/__init__.py b/tests/generic_config_updater/add_cluster/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/generic_config_updater/add_cluster/acl/acl_rule_src_dst_port.json b/tests/generic_config_updater/add_cluster/acl/acl_rule_src_dst_port.json new file mode 100644 index 00000000000..12a21a03aed --- /dev/null +++ b/tests/generic_config_updater/add_cluster/acl/acl_rule_src_dst_port.json @@ -0,0 +1,44 @@ +{ + "acl": { + "acl-sets": { + "acl-set": { + "L3_TRANSPORT_TEST": { + "acl-entries": { + "acl-entry": { + "100": { + "actions": { + "config": { + "forwarding-action": "ACCEPT" + } + }, + "config": { + "sequence-id": 100 + }, + "transport": { + "config": { + "source-port": "5000" + } + } + }, + "200": { + "actions": { + "config": { + "forwarding-action": "DROP" + } + }, + "config": { + "sequence-id": 200 + }, + "transport": { + "config": { + "destination-port": "8080" + } + } + } + } + } + } + } + } + } +} diff --git a/tests/generic_config_updater/add_cluster/conftest.py b/tests/generic_config_updater/add_cluster/conftest.py new file mode 100644 index 00000000000..d1311a43a9f --- /dev/null +++ b/tests/generic_config_updater/add_cluster/conftest.py @@ -0,0 +1,74 @@ +import logging +import pytest +from tests.common.gu_utils import create_checkpoint, delete_checkpoint, rollback_or_reload +from tests.common.gu_utils import restore_backup_test_config, save_backup_test_config + +logger = logging.getLogger(__name__) + + +# ----------------------------- +# Fixtures that return random values for selected asic namespace, neighbors and cfg data for these selections +# ----------------------------- + +@pytest.fixture(scope="module") +def enum_rand_one_asic_namespace(enum_rand_one_frontend_asic_index): + return None if enum_rand_one_frontend_asic_index is None else 'asic{}'.format(enum_rand_one_frontend_asic_index) + + +@pytest.fixture(scope="module") +def config_facts(duthosts, enum_downstream_dut_hostname, enum_rand_one_asic_namespace): + duthost = duthosts[enum_downstream_dut_hostname] + return duthost.config_facts( + host=duthost.hostname, source="running", namespace=enum_rand_one_asic_namespace + )['ansible_facts'] + + +@pytest.fixture(scope="module") +def config_facts_localhost(duthosts, enum_downstream_dut_hostname): + duthost = duthosts[enum_downstream_dut_hostname] + return duthost.config_facts(host=duthost.hostname, source="running", namespace=None)['ansible_facts'] + + +@pytest.fixture(scope="module") +def mg_facts(duthosts, enum_downstream_dut_hostname, enum_rand_one_asic_namespace, tbinfo): + duthost = duthosts[enum_downstream_dut_hostname] + return duthost.get_extended_minigraph_facts(tbinfo, namespace=enum_rand_one_asic_namespace) + + +@pytest.fixture(scope="module") +def rand_bgp_neigh_ip_name(config_facts): + '''Returns a random bgp neighbor ip, name from the namespace''' + bgp_neighbors = config_facts["BGP_NEIGHBOR"] + random_bgp_neigh_ip = list(bgp_neighbors.keys())[0] + random_bgp_neigh_name = config_facts['BGP_NEIGHBOR'][random_bgp_neigh_ip]['name'] + logger.info("rand_bgp_neigh_ip_name : {}, {} " + .format(random_bgp_neigh_ip, random_bgp_neigh_name)) + return random_bgp_neigh_ip, random_bgp_neigh_name + + +# ----------------------------- +# Setup Fixtures +# ----------------------------- + +@pytest.fixture(scope="module", autouse=True) +def setup_env(duthosts, rand_one_dut_front_end_hostname): + """ + Setup/teardown fixture for add cluster test cases. + Args: + duthosts: list of DUTs. + rand_one_dut_front_end_hostname: A random linecard. + """ + + duthost = duthosts[rand_one_dut_front_end_hostname] + create_checkpoint(duthost) + save_backup_test_config(duthost, file_postfix="{}_before_add_cluster_test".format(duthost.hostname)) + + yield + + restore_backup_test_config(duthost, file_postfix="{}_before_add_cluster_test".format(duthost.hostname), + config_reload=False) + try: + logger.info("{}:Rolling back to original checkpoint".format(duthost.hostname)) + rollback_or_reload(duthost) + finally: + delete_checkpoint(duthost) diff --git a/tests/generic_config_updater/add_cluster/helpers.py b/tests/generic_config_updater/add_cluster/helpers.py new file mode 100644 index 00000000000..97599f8ba97 --- /dev/null +++ b/tests/generic_config_updater/add_cluster/helpers.py @@ -0,0 +1,468 @@ +import json +import logging +import random +import re +import time + +import requests +import pytest +import ptf.testutils as testutils +import ptf.mask as mask +import ptf.packet as packet +from tests.common.gu_utils import apply_patch, delete_tmpfile, expect_op_success, generate_tmpfile +from tests.common.helpers.assertions import pytest_assert +from tests.common.snappi_tests.common_helpers import clear_counters, get_queue_count_all_prio +from tests.common.utilities import wait_until + +logger = logging.getLogger(__name__) + + +# ----------------------------- +# Static Route Helper Functions +# ----------------------------- + +def get_exabgp_port_for_neighbor(tbinfo, neigh_name, exabgp_base_port=5000): + offset = tbinfo['topo']['properties']['topology']['VMs'][neigh_name]['vm_offset'] + exabgp_port = exabgp_base_port + offset + return exabgp_port + + +def change_route(operation, ptfip, route, nexthop, port, aspath): + url = "http://%s:%d" % (ptfip, port) + data = { + "command": "%s route %s next-hop %s as-path [ %s ]" % (operation, route, nexthop, aspath)} + r = requests.post(url, data=data) + assert r.status_code == 200 + + +def add_static_route(tbinfo, neigh_ip, exabgp_port, ip, mask='32', aspath=65500, nhipv4='10.10.246.254'): + common_config = tbinfo['topo']['properties']['configuration_properties'].get('common', {}) + ptf_ip = tbinfo['ptf_ip'] + dst_prefix = ip + '/' + mask + nexthop = common_config.get('nhipv4', nhipv4) + logger.info( + "Announcing route: ptf_ip={} dst_prefix={} nexthop={} exabgp_port={} aspath={} via neighbor {}".format( + ptf_ip, dst_prefix, nexthop, exabgp_port, aspath, neigh_ip) + ) + change_route('announce', ptf_ip, dst_prefix, nexthop, exabgp_port, aspath) + + +def clear_static_route(tbinfo, duthost, ip, nhipv4='10.10.246.254'): + config_facts_localhost = duthost.config_facts(host=duthost.hostname, source='running', + verbose=False, namespace=None + )['ansible_facts'] + num_asic = duthost.facts.get('num_asic') + for asic_index in range(num_asic): + output = duthost.shell("sudo ip netns exec asic{} show ip route | grep {}" + .format(asic_index, ip))['stdout'] + ip_address = re.search(r'via (\d+\.\d+\.\d+\.\d+)', output) + if ip_address: + ip_address = ip_address.group(1) + bgp_neigh_name = config_facts_localhost['BGP_NEIGHBOR'][ip_address]['name'] + exabgp_port = get_exabgp_port_for_neighbor(tbinfo, bgp_neigh_name) + remove_static_route(tbinfo, ip_address, exabgp_port, ip=ip, nhipv4=nhipv4) + wait_until(10, 1, 0, verify_routev4_existence, duthost, asic_index, ip, should_exist=False) + + +def remove_static_route(tbinfo, neigh_ip, exabgp_port, ip, mask='32', aspath=65500, nhipv4='10.10.246.254'): + common_config = tbinfo['topo']['properties']['configuration_properties'].get('common', {}) + ptf_ip = tbinfo['ptf_ip'] + dst_prefix = ip + '/' + mask + nexthop = common_config.get('nhipv4', nhipv4) + logger.info( + "Withdrawing route: ptf_ip={} dst_prefix={} nexthop={} exabgp_port={} aspath={} via neighbor {}".format( + ptf_ip, dst_prefix, nexthop, exabgp_port, aspath, neigh_ip + ) + ) + change_route('withdraw', ptf_ip, dst_prefix, nexthop, exabgp_port, aspath) + + +def verify_routev4_existence(duthost, asic_id, ip, should_exist=True): + cur_ipv4_routes = duthost.asic_instance(asic_id).command("ip -4 route")['stdout'] + if ip in cur_ipv4_routes: + logger.info("{}:Verifying route {} existence || Found=True || Expected={}.".format(duthost, ip, should_exist)) + return True if should_exist else False + else: + logger.info("{}:Verifying route {} existence || Found=False || Expected={}.".format(duthost, ip, should_exist)) + return False if should_exist else True + + +# ----------------------------- +# Apply Patch Related Helper Functions +# ----------------------------- + +def add_content_to_patch_file(json_data, patch_file): + logger.info("Adding extra content to patch file = {}".format(patch_file)) + + try: + with open(patch_file, "r") as file: + existing_content = json.load(file) + except (FileNotFoundError, json.JSONDecodeError): + existing_content = [] + + if isinstance(json_data, str): + try: + json_data = json.loads(json_data) + except json.JSONDecodeError: + logger.error("Invalid JSON format in json_data") + raise ValueError("json_data must be a valid JSON list or dictionary") + + if isinstance(existing_content, list) and isinstance(json_data, list): + existing_content.extend(json_data) + elif isinstance(existing_content, dict) and isinstance(json_data, dict): + existing_content.update(json_data) + else: + raise ValueError("add_content_to_patch_file: Mismatched types: Cannot merge {} with {}".format( + type(existing_content).__name__, type(json_data).__name__ + )) + + with open(patch_file, "w") as file: + json.dump(existing_content, file, indent=4) + + +def change_interface_admin_state_for_namespace(config_facts, + duthost, + namespace, + status=None, + apply=True, + verify=True, + patch_file=""): + """ + Applies a patch to change the administrative status (up/down) of interfaces for a specific namespace + on the DUT host. + + Applies changes at configuration path: + - //PORT//admin_status + + This function updates the administrative state (enabled/disabled) of interfaces within the specified namespace + on the DUT host by applying a patch. It also offers optional verification of the changes. + + Args: + config_facts (dict): Configuration facts from the DUT host, containing the current state of the configuration. + duthost (object): DUT host object on which the patch will be applied. + namespace (str): The namespace whose interfaces should have their administrative state modified. + status (str, optional): The desired administrative state of the interfaces ('up' or 'down'). If not provided, + no state change is applied. Defaults to None. + verify (bool, optional): If True, verifies the changes after applying the patch. Defaults to True. + + Returns: + None + + Raises: + Exception: If the patch or verification fails. + """ + + pytest_assert(status, "Test didn't provided the admin status value to change to.") + + logger.info("{}: Changing admin status for local interfaces to {} for ASIC namespace {}".format( + duthost.hostname, status, namespace) + ) + json_namespace = '' if namespace is None else '/' + namespace + json_patch = [] + + # find all the interfaces that are active based on configuration + up_interfaces = [] + for key, _value in config_facts.get("INTERFACE", {}).items(): + if re.compile(r'^Ethernet\d{1,3}$').match(key): + up_interfaces.append(key) + for portchannel in config_facts.get("PORTCHANNEL_MEMBER", {}): + for key, _value in config_facts.get("PORTCHANNEL_MEMBER", {}).get(portchannel, {}).items(): + up_interfaces.append(key) + logger.info("Up interfaces for this namespace:{}".format(up_interfaces)) + + for interface in up_interfaces: + json_patch.append({ + "op": "add", + "path": "{}/PORT/{}/admin_status".format(json_namespace, interface), + "value": status + }) + + if apply: + + tmpfile = generate_tmpfile(duthost) + + try: + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + if verify is True: + logger.info("{}: Verifying interfaces status is {}.".format(duthost.hostname, status)) + pytest_assert(check_interface_status(duthost, namespace, up_interfaces, exp_status=status), + "Interfaces failed to update admin status to {}'".format(status)) + finally: + delete_tmpfile(duthost, tmpfile) + else: + add_content_to_patch_file(json.dumps(json_patch, indent=4), patch_file) + + +# ----------------------------- +# Helper Functions - Interfaces, Config +# ----------------------------- + +def check_interface_status(duthost, namespace, interface_list, exp_status='up'): + """ + Verifies if all interfaces for one namespace are the expected status + Args: + duthost: DUT host object under test + namespace: Namespace to verify + interface_list: The list of interfaces to verify + exp_status: Expected status for all the interfaces + """ + for interface in interface_list: + cmds = "show interface status {} -n {}".format(interface, namespace) + output = duthost.shell(cmds) + pytest_assert(not output['rc']) + status_data = output["stdout_lines"] + field_index = status_data[0].split().index("Admin") + for line in status_data: + interface_status = line.strip() + pytest_assert(len(interface_status) > 0, "Failed to read line {}".format(line)) + if interface_status.startswith(interface): + status = re.split(r" {2,}", interface_status)[field_index] + if status != exp_status: + logger.error("Found interface {} in non-expected state {}. Line output: {}".format( + interface, interface_status, line)) + return False + return True + + +def get_cfg_info_from_dut(duthost, path, enum_rand_one_asic_namespace): + """ + Returns the running configuration for a given configuration path within a namespace. + """ + dict_info = None + namespace_prefix = '' if enum_rand_one_asic_namespace is None else '-n ' + enum_rand_one_asic_namespace + raw_output = duthost.command( + "sudo sonic-cfggen {} -d --var-json {}".format( + namespace_prefix, path) + )["stdout"] + try: + dict_info = json.loads(raw_output) + except json.JSONDecodeError: + dict_info = None + + if not isinstance(dict_info, dict): + print("Expected a dictionary, but got:", type(dict_info)) + dict_info = None + return dict_info + + +def get_active_interfaces(config_facts): + """ + Finds all the active interfaces based on running configuration. + """ + active_interfaces = [] + for key, _value in config_facts.get("INTERFACE", {}).items(): + if re.compile(r'^Ethernet\d{1,3}$').match(key): + active_interfaces.append(key) + for portchannel in config_facts.get("PORTCHANNEL_MEMBER", {}): + for key, _value in config_facts.get("PORTCHANNEL_MEMBER", {}).get(portchannel, {}).items(): + active_interfaces.append(key) + logger.info("Active interfaces for this namespace:{}".format(active_interfaces)) + return active_interfaces + + +def select_random_active_interface(duthost, namespace): + """ + Finds all the active interfaces based on status in duthost and returns a random selected. + """ + interfaces = duthost.get_interfaces_status(namespace) + active_interfaces = [] + for interface_name, interface_info in list(interfaces.items()): + if interface_name.startswith('Ethernet') \ + and interface_info.get('oper') == 'up' \ + and interface_info.get('admin') == 'up': + active_interfaces.append(interface_name) + return random.choice(active_interfaces) + + +# ----------------------------- +# ACL Helper Functions and Variables +# ----------------------------- + +def acl_asic_shell_wrappper(duthost, cmd, asic=''): + def run_cmd(host, command): + if isinstance(command, list): + for cm in command: + host.shell(cm) + else: + host.shell(command) + + if duthost.is_multi_asic: + asics = [duthost.asics[int(asic.replace("asic", ""))]] if asic else duthost.asics + + for asichost in asics: + ns_cmd = ["{} {}".format(asichost.ns_arg, cm) for cm in (cmd if isinstance(cmd, list) else [cmd])] + run_cmd(asichost, ns_cmd) + else: + run_cmd(duthost, cmd) + + +def remove_dataacl_table_single_dut(table_name, duthost): + lines = duthost.shell(cmd="show acl table {}".format(table_name))['stdout_lines'] + data_acl_existing = False + for line in lines: + if table_name in line: + data_acl_existing = True + break + if data_acl_existing: + # Remove DATAACL + logger.info("{} Removing ACL table {}".format(duthost.hostname, table_name)) + cmds = [ + "config acl remove table {}".format(table_name), + "config save -y" + ] + acl_asic_shell_wrappper(duthost, cmds) + + +def get_cacl_tables(duthost, ip_netns_namespace_prefix): + """Get acl control plane tables + """ + cmds = "{} show acl table | grep -w CTRLPLANE | awk '{{print $1}}'".format(ip_netns_namespace_prefix) + + output = duthost.shell(cmds) + pytest_assert(not output['rc'], "'{}' failed with rc={}".format(cmds, output['rc'])) + cacl_tables = output['stdout'].splitlines() + return cacl_tables + + +# ----------------------------- +# Data Traffic Helper Functions +# ----------------------------- + +def send_and_verify_traffic( + tbinfo, + src_duthost, + dst_duthost, + src_asic_index, + dst_asic_index, + ptfadapter, + ptf_sport=None, + ptf_dst_ports=None, + ptf_dst_interfaces=None, + src_ip='30.0.0.10', + dst_ip='50.0.2.2', + count=1, + dscp=None, + sport=0x1234, + dport=0x50, + flags=0x10, + verify=True, + expect_error=False + ): + """ + Helper function to send and verify data traffic via PTF framework. + """ + + src_asic_namespace = None if src_asic_index is None else 'asic{}'.format(src_asic_index) + dst_asic_namespace = None if dst_asic_index is None else 'asic{}'.format(dst_asic_index) + router_mac = src_duthost.asic_instance(src_asic_index).get_router_mac() + src_mg_facts = src_duthost.get_extended_minigraph_facts(tbinfo, src_asic_namespace) + dst_mg_facts = dst_duthost.get_extended_minigraph_facts(tbinfo, dst_asic_namespace) + + # port from ptf + if not ptf_sport: + ptf_src_ports = list(src_mg_facts["minigraph_ptf_indices"].values()) + ptf_sport = random.choice(ptf_src_ports) + if not ptf_dst_ports: + ptf_dst_ports = list(set(dst_mg_facts["minigraph_ptf_indices"].values())) + if not ptf_dst_interfaces: + ptf_dst_interfaces = list(set(dst_mg_facts["minigraph_ptf_indices"].keys())) + + # clear counters + clear_counters(dst_duthost, namespace=dst_asic_namespace) + + # Create pkt + pkt = testutils.simple_tcp_packet( + eth_src=ptfadapter.dataplane.get_mac(0, ptf_sport), + eth_dst=router_mac, + ip_src=src_ip, + ip_dst=dst_ip, + ip_ttl=64, + ip_dscp=dscp, + tcp_sport=sport, + tcp_dport=dport, + tcp_flags=flags + ) + logging.info("Packet created: {}".format(pkt)) + + # Create exp packet for verification + exp_pkt = pkt.copy() + exp_pkt = mask.Mask(exp_pkt) + exp_pkt.set_do_not_care_scapy(packet.Ether, 'dst') + exp_pkt.set_do_not_care_scapy(packet.Ether, 'src') + exp_pkt.set_do_not_care_scapy(packet.IP, 'ttl') + exp_pkt.set_do_not_care_scapy(packet.IP, 'chksum') + + # Send packet + ptfadapter.dataplane.flush() + testutils.send(ptfadapter, ptf_sport, pkt, count=count) + + # Verify packet count from ptfadapter + if verify: + if expect_error: + with pytest.raises(AssertionError): + testutils.verify_packet_any_port(ptfadapter, exp_pkt, ports=ptf_dst_ports) + else: + testutils.verify_packet_any_port(ptfadapter, exp_pkt, ports=ptf_dst_ports) + + # verify queue counters + if dscp: + logging.info("Verifying queue counters for dscp {}.".format(dscp)) + exp_prio = 'prio_{}'.format(dscp) + retry_count = 3 + retry_int = 5 + + def get_counters(): + counter_exp_prio = 0 + counter_rest_prio = 0 + for interface in ptf_dst_interfaces: + if interface.startswith('Ethernet-IB'): + continue + interface_queue_count_dict = get_queue_count_all_prio(dst_duthost, interface) + for prio, prio_counter in interface_queue_count_dict[dst_duthost.hostname][interface].items(): + if prio != exp_prio: + counter_rest_prio += prio_counter + else: + counter_exp_prio += prio_counter + return counter_exp_prio, counter_rest_prio + + for attempt in range(1, retry_count + 1): + time.sleep(retry_int) + counter_exp_prio, counter_rest_prio = get_counters() + + if expect_error: + if counter_exp_prio == 0 and counter_rest_prio == 0: + logging.info(f"Attempt {attempt}: Expected counters verified (both zero).") + break + else: + if counter_exp_prio == count and counter_rest_prio == 0: + logging.info(f"Attempt {attempt}: Expected counters verified successfully.") + break + + if attempt < retry_count: + logging.warning( + f"Attempt {attempt}: Counters not as expected. Retrying in {retry_int}s..." + ) + else: + logging.error("Max retries reached. Failure in queue counter verification.") + + if expect_error: + pytest_assert( + counter_exp_prio == 0 and counter_rest_prio == 0, + 'Found unexpected queue counter values.\n \ + Prio{} Queues Expected: 0 - Found:{}.\n \ + Rest Prio Queues Expected: 0 - Found:{}.'.format(dscp, counter_exp_prio, counter_rest_prio) + ) + else: + pytest_assert( + counter_exp_prio == count and counter_rest_prio == 0, + 'Found unexpected queue counter values.\n \ + Prio{} Queues Expected:{} - Found:{}.\n \ + Rest Prio Queues Expected: 0 - Found:{}.'.format(dscp, count, counter_exp_prio, counter_rest_prio) + ) + logger.info("Success queue counter verification - \ + Prio{} Queues Counter:{} - \ + Rest Prio Queues Counter:{}.".format( + dscp, counter_exp_prio, counter_rest_prio + ) + ) diff --git a/tests/generic_config_updater/add_cluster/test_add_cluster.py b/tests/generic_config_updater/add_cluster/test_add_cluster.py new file mode 100644 index 00000000000..8855cde995b --- /dev/null +++ b/tests/generic_config_updater/add_cluster/test_add_cluster.py @@ -0,0 +1,1161 @@ +import logging +import pytest +from tests.common.helpers.assertions import pytest_assert +from tests.common.utilities import wait_until +from tests.common.plugins.allure_wrapper import allure_step_wrapper as allure +from tests.common.platform.interface_utils import check_interface_status_of_up_ports +from tests.common.config_reload import config_reload +from tests.common.gu_utils import delete_tmpfile, expect_op_success, generate_tmpfile +from tests.common.gu_utils import apply_patch +from tests.generic_config_updater.add_cluster.helpers import add_static_route, \ + clear_static_route, get_active_interfaces, get_cfg_info_from_dut, \ + get_exabgp_port_for_neighbor, remove_dataacl_table_single_dut, remove_static_route, \ + send_and_verify_traffic, verify_routev4_existence + +pytestmark = [ + pytest.mark.topology("t2") + ] + +logger = logging.getLogger(__name__) +allure.logger = logger + + +# ----------------------------- +# Attributes used by test for static route, acl config +# ----------------------------- + +EXABGP_BASE_PORT = 5000 +NHIPV4 = '10.10.246.254' +STATIC_DST_IP = '1.1.1.1' + +ACL_TABLE_NAME = "L3_TRANSPORT_TEST" +ACL_TABLE_STAGE_EGRESS = "egress" +ACL_TABLE_TYPE_L3 = "L3" +ACL_RULE_FILE_PATH = "generic_config_updater/add_cluster/acl/acl_rule_src_dst_port.json" +ACL_RULE_DST_FILE = "/tmp/test_add_cluster_acl_rule.json" +ACL_RULE_SKIP_VERIFICATION_LIST = [""] + +# ----------------------------- +# Helper functions that validate apply-patch changes +# ----------------------------- + + +def verify_bgp_peers_removed_from_asic(duthost, namespace): + logger.info("{}: Verifying bgp_neighbors info is removed.".format(duthost.hostname)) + cur_bgp_neighbors = get_cfg_info_from_dut(duthost, "BGP_NEIGHBOR", namespace) + cur_device_neighbor = get_cfg_info_from_dut(duthost, "DEVICE_NEIGHBOR", namespace) + cur_device_neighbor_metadata = get_cfg_info_from_dut(duthost, "DEVICE_NEIGHBOR_METADATA", namespace) + pytest_assert(not cur_bgp_neighbors, + "Bgp neighbors info removal via apply-patch failed." + ) + pytest_assert(not cur_device_neighbor, + "Device neighbor info removal via apply-patch failed." + ) + pytest_assert(not cur_device_neighbor_metadata, + "Device neighbor metadata info removal via apply-patch failed." + ) + + +# ----------------------------- +# Helper functions that modify configuration via apply-patch +# ----------------------------- +def remove_cluster_via_sonic_db_cli(config_facts, + config_facts_localhost, + mg_facts, + duthost, + enum_rand_one_asic_namespace, + cli_namespace_prefix): + """ + Remove cluster information directly from CONFIG_DB using sonic-db-cli commands, + bypassing YANG validation but safely and persistently. + + Performs same cleanup as apply_patch_remove_cluster: + - ACL_TABLE + - BGP_NEIGHBOR + - DEVICE_NEIGHBOR + - DEVICE_NEIGHBOR_METADATA + - PORTCHANNEL + - PORTCHANNEL_INTERFACE + - PORTCHANNEL_MEMBER + - INTERFACE + - BUFFER_PG + - CABLE_LENGTH + - PORT_QOS_MAP + - PORT + """ + + json_namespace = '' if enum_rand_one_asic_namespace is None else enum_rand_one_asic_namespace + logger.info(f"Starting cluster removal for ASIC namespace: {json_namespace}") + + active_interfaces = get_active_interfaces(config_facts) + success = True + + def run_and_check(ns, cmd, desc): + """Run a shell command on DUT and check for success.""" + logger.info(f"[{ns}] {desc}: {cmd}") + res = duthost.shell(cmd, module_ignore_errors=True) + if res["rc"] != 0: + logger.warning(f"[WARN] Command failed: {cmd}\nstdout: {res['stdout']}\nstderr: {res['stderr']}") + return False + return True + + ###################### + # ASIC NAMESPACE + ###################### + if json_namespace: + logger.info(f"Cleaning up ASIC namespace: {json_namespace}") + + # BGP_NEIGHBOR, DEVICE_NEIGHBOR, DEVICE_NEIGHBOR_METADATA + for table in ["BGP_NEIGHBOR", "DEVICE_NEIGHBOR", "DEVICE_NEIGHBOR_METADATA"]: + cmd = f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB keys '{table}*' \ + | xargs -r -n1 sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB del" + run_and_check(json_namespace, cmd, f"Clearing table {table}") + + # INTERFACE + asic_interface_keys = [] + for interface_key in config_facts["INTERFACE"].keys(): + if interface_key.startswith("Ethernet-Rec"): + continue + for key, _value in config_facts["INTERFACE"][interface_key].items(): + asic_interface_keys.append(interface_key + '|' + key) + asic_interface_keys.append(interface_key) + for iface in asic_interface_keys: + run_and_check(json_namespace, + f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB del 'INTERFACE|{iface}'", + f"Deleting INTERFACE {iface}") + + # PORTCHANNEL_INTERFACE, PORTCHANNEL_MEMBER + for table in ["PORTCHANNEL_INTERFACE", "PORTCHANNEL_MEMBER"]: + cmd = f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB keys '{table}*' \ + | xargs -r -n1 sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB del" + run_and_check(json_namespace, cmd, f"Clearing table {table}") + + # ACL + for acl_table in ["DATAACL", "EVERFLOW", "EVERFLOWV6"]: + cmd = f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB hdel 'ACL_TABLE|{acl_table}' ports@" + run_and_check(json_namespace, cmd, f"Removing ACL_TABLE {acl_table} ports") + + # PORTCHANNEL + cmd = f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB keys 'PORTCHANNEL*' \ + | xargs -r -n1 sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB del" + run_and_check(json_namespace, cmd, "Clearing table PORTCHANNEL") + + # CABLE_LENGTH + initial = config_facts["CABLE_LENGTH"]["AZURE"] + lowest = min(int(v.rstrip("m")) for v in initial.values()) + for iface in active_interfaces: + run_and_check(json_namespace, + f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB hset \ + 'CABLE_LENGTH|AZURE' {iface} '{lowest}m'", + f"Set cable length for {iface}" + ) + # PORT + for iface in active_interfaces: + run_and_check(json_namespace, + f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB hset 'PORT|{iface}' admin_status down", + f"Set {iface} admin down") + # BUFFER_PG + cmd = f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB keys 'BUFFER_PG*' \ + | xargs -r -n1 sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB del" + run_and_check(json_namespace, cmd, "Clearing table BUFFER_PG") + + # PORT_QOS_MAP + for iface in active_interfaces: + run_and_check(json_namespace, + f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB del 'PORT_QOS_MAP|{iface}'", + f"Deleting PORT_QOS_MAP for {iface}") + + ###################### + # LOCALHOST NAMESPACE + ###################### + logger.info("Cleaning up localhost namespace") + # BGP_NEIGHBOR + for entry in config_facts["BGP_NEIGHBOR"].keys(): + run_and_check("localhost", + f"sudo sonic-db-cli CONFIG_DB del 'BGP_NEIGHBOR|{entry}'", + f"Deleting localhost BGP_NEIGHBOR {entry}") + # DEVICE_NEIGHBOR_METADATA + for entry in config_facts["DEVICE_NEIGHBOR_METADATA"].keys(): + run_and_check("localhost", + f"sudo sonic-db-cli CONFIG_DB del 'DEVICE_NEIGHBOR_METADATA|{entry}'", + f"Deleting localhost DEVICE_NEIGHBOR_METADATA {entry}") + # INTERFACE + localhost_interface_keys = [] + for key in asic_interface_keys: + if key.startswith('Ethernet-Rec'): + continue + parts = key.split('|') + key_to_remove = key + if len(parts) == 2: + port = parts[0] + alias = mg_facts['minigraph_port_name_to_alias_map'].get(port, port) + key_to_remove = "{}|{}".format(alias, parts[1]) + else: + key_to_remove = mg_facts['minigraph_port_name_to_alias_map'].get(key, key) + localhost_interface_keys.append(key_to_remove) + for iface in localhost_interface_keys: + run_and_check("localhost", + f"sudo sonic-db-cli CONFIG_DB del 'INTERFACE|{iface}'", + f"Deleting localhost INTERFACE {iface}") + # PORTCHANNEL_INTERFACE + for entry in config_facts.get("PORTCHANNEL_INTERFACE", {}).keys(): + run_and_check("localhost", + f"sudo sonic-db-cli CONFIG_DB del 'PORTCHANNEL_INTERFACE|{entry}'", + f"Deleting localhost PORTCHANNEL_INTERFACE {entry}") + # PORTCHANNEL_MEMBER + pc_keys = config_facts.get("PORTCHANNEL", {}).keys() + localhost_pc_member_dict = config_facts_localhost.get("PORTCHANNEL_MEMBER", {}) + localhost_pc_member_keys = [] + for pc_key in pc_keys: + if pc_key in localhost_pc_member_dict: + for key, _value in localhost_pc_member_dict[pc_key].items(): + key_to_remove = pc_key + '|' + key + localhost_pc_member_keys.append(key_to_remove) + for entry in localhost_pc_member_keys: + run_and_check("localhost", + f"sudo sonic-db-cli CONFIG_DB del 'PORTCHANNEL_MEMBER|{entry}'", + f"Deleting localhost PORTCHANNEL_MEMBER {entry}") + # ACL localhost - need to remove only the entries from asic namespace + for acl_table in ["DATAACL", "EVERFLOW", "EVERFLOWV6"]: + run_and_check("localhost", + f"sudo sonic-db-cli CONFIG_DB hdel 'ACL_TABLE|{acl_table}' ports@", + f"Removing localhost ACL_TABLE {acl_table} ports") + # PORTCHANNEL + for entry in config_facts("PORTCHANNEL", {}).keys(): + run_and_check("localhost", + f"sudo sonic-db-cli CONFIG_DB del 'PORTCHANNEL|{entry}'", + f"Deleting localhost PORTCHANNEL {entry}") + + # Partial Verification + logger.info("Verifying that asic tables were cleared...") + tables_to_check = [ + "BGP_NEIGHBOR", "DEVICE_NEIGHBOR", "DEVICE_NEIGHBOR_METADATA", + "PORTCHANNEL_INTERFACE", "PORTCHANNEL_MEMBER", "PORTCHANNEL" + ] + for table in tables_to_check: + cmd = f"sudo sonic-db-cli {cli_namespace_prefix} CONFIG_DB keys '{table}*'" + res = duthost.shell(cmd, module_ignore_errors=True) + if res["stdout"].strip(): + logger.warning(f"{table} still contains entries: {res['stdout']}") + success = False + else: + logger.info(f"{table} is empty") + + if success: + logger.info("Cluster removal completed successfully.") + else: + logger.warning("Cluster removal incomplete — verification failure.") + + +def apply_patch_remove_cluster(config_facts, + config_facts_localhost, + mg_facts, + duthost, + enum_rand_one_asic_namespace, + cli_namespace_prefix): + """ + Apply patch to remove cluster information for a given ASIC namespace. + + Changes are perfomed to below tables: + + ACL_TABLE + BGP_NEIGHBOR + DEVICE_NEIGHBOR + DEVICE_NEIGHBOR_METADATA + PORTCHANNEL + PORTCHANNEL_INTERFACE + PORTCHANNEL_MEMBER + INTERFACE + BUFFER_PG + CABLE_LENGTH + PORT + PORT_QOS_MAP + + """ + + logger.info("Removing cluster for namespace {} via apply-patch.".format(enum_rand_one_asic_namespace)) + + ###################### + # ASIC NAMESPACE + ###################### + json_patch_asic = [] + logger.info("{}: Removing cluster info for namespace {}".format(duthost.hostname, enum_rand_one_asic_namespace)) + json_namespace = '' if enum_rand_one_asic_namespace is None else '/' + enum_rand_one_asic_namespace + + asic_paths_list = [] + + # find active ports + active_interfaces = get_active_interfaces(config_facts) + + # W/A: TABLE:ACL_TABLE removing whole table instead of detaching ports + # https://github.com/sonic-net/sonic-buildimage/issues/24295 + + # op: remove + json_patch_asic = [ + { + "op": "remove", + "path": f"{json_namespace}/ACL_TABLE/DATAACL" + }, + { + "op": "remove", + "path": f"{json_namespace}/ACL_TABLE/EVERFLOW" + }, + { + "op": "remove", + "path": f"{json_namespace}/ACL_TABLE/EVERFLOWV6" + }, + { + "op": "remove", + "path": f"{json_namespace}/BGP_NEIGHBOR" + }, + { + "op": "remove", + "path": f"{json_namespace}/DEVICE_NEIGHBOR" + }, + { + "op": "remove", + "path": f"{json_namespace}/DEVICE_NEIGHBOR_METADATA" + }, + { + "op": "remove", + "path": f"{json_namespace}/BUFFER_PG" + } + ] + + if 'PORTCHANNEL' in config_facts: + json_patch_asic.append( + { + "op": "remove", + "path": f"{json_namespace}/PORTCHANNEL_MEMBER" + } + ) + json_patch_asic.append( + { + "op": "remove", + "path": f"{json_namespace}/PORTCHANNEL_INTERFACE" + } + ) + + # table INTERFACE + if 'INTERFACE' in config_facts: + asic_interface_dict = config_facts["INTERFACE"] + asic_interface_keys = [] + asic_interface_ip_prefix_keys = [] + for interface_key in asic_interface_dict.keys(): + if interface_key.startswith("Ethernet-Rec"): + continue + for key, _value in asic_interface_dict[interface_key].items(): + key_to_remove = interface_key + '|' + key.replace("/", "~1") + asic_interface_ip_prefix_keys.append(key_to_remove) + asic_interface_keys.append(interface_key) + + for key in asic_interface_ip_prefix_keys: + asic_paths_list.append(f"{json_namespace}/INTERFACE/" + key) + + for path in asic_paths_list: + json_patch_asic.append({ + "op": "remove", + "path": path + }) + + # table PORT_QOS_MAP changes + for interface in active_interfaces: + json_patch_asic.append({ + "op": "remove", + "path": "{}/PORT_QOS_MAP/{}".format(json_namespace, interface) + }) + + # table PORT changes + for interface in active_interfaces: + json_patch_asic.append({ + "op": "add", + "path": "{}/PORT/{}/admin_status".format(json_namespace, interface), + "value": "down" + }) + + # table CABLE_LENGTH changes + initial_cable_length_table = config_facts["CABLE_LENGTH"]["AZURE"] + cable_length_values = [int(v.rstrip("m")) for v in initial_cable_length_table.values()] + lowest = min(cable_length_values) + for interface in active_interfaces: + json_patch_asic.append({ + "op": "add", + "path": "{}/CABLE_LENGTH/AZURE/{}".format(json_namespace, interface), + "value": f"{lowest}m" + }) + ###################### + # LOCALHOST NAMESPACE + ###################### + json_patch_localhost = [] + logger.info("{}: Removing cluster info for namespace localhost".format(duthost.hostname)) + + # INTERFACE TABLE: in localhost replace the interface name with the interface alias + # INTERFACE ip-prefix + if 'INTERFACE' in config_facts: + localhost_ip_prefix_interface_keys = [] + for key in asic_interface_ip_prefix_keys: + parts = key.split('|') + port = parts[0] + alias = mg_facts['minigraph_port_name_to_alias_map'].get(port, port) + key_to_remove = "{}|{}".format(alias, parts[1]) + key_to_remove = key_to_remove.replace("/", "~1") + localhost_ip_prefix_interface_keys.append(key_to_remove) + # INTERFACE name + localhost_interface_keys = [] + for key in asic_interface_keys: + key_to_remove = mg_facts['minigraph_port_name_to_alias_map'].get(key, key) + key_to_remove = key_to_remove.replace("/", "~1") + localhost_interface_keys.append(key_to_remove) + + # PORTCHANNEL_MEMBER keys + if 'PORTCHANNEL' in config_facts: + pc_keys = config_facts.get("PORTCHANNEL", {}).keys() + + localhost_pc_member_dict = config_facts_localhost.get("PORTCHANNEL_MEMBER", {}) + localhost_pc_member_keys = [] + for pc_key in pc_keys: + if pc_key in localhost_pc_member_dict: + for key, _value in localhost_pc_member_dict[pc_key].items(): + key_to_remove = pc_key + '|' + key.replace("/", "~1") + localhost_pc_member_keys.append(key_to_remove) + # PORTCHANNEL_INTERFACE keys + localhost_pc_interface_dict = config_facts_localhost.get("PORTCHANNEL_INTERFACE", {}) + localhost_pc_interface_keys = [] + for pc_key in pc_keys: + if pc_key in localhost_pc_interface_dict: + for key, _value in localhost_pc_interface_dict[pc_key].items(): + key_to_remove = pc_key + '|' + key.replace("/", "~1") + localhost_pc_interface_keys.append(key_to_remove) + localhost_pc_interface_keys.append(pc_key) + + # ACL TABLE + acl_ports_localhost = config_facts_localhost["ACL_TABLE"]["DATAACL"]["ports"] + acl_ports_asic = config_facts["ACL_TABLE"]["DATAACL"]["ports"] + acl_ports_localhost_post_removal = [p for p in acl_ports_localhost if p not in acl_ports_asic] + if acl_ports_localhost_post_removal: + json_patch_localhost = [ + { + "op": "add", + "path": "/localhost/ACL_TABLE/DATAACL/ports", + "value": acl_ports_localhost_post_removal + }, + { + "op": "add", + "path": "/localhost/ACL_TABLE/EVERFLOW/ports", + "value": acl_ports_localhost_post_removal + }, + { + "op": "add", + "path": "/localhost/ACL_TABLE/EVERFLOWV6/ports", + "value": acl_ports_localhost_post_removal + } + ] + localhost_paths_list = [] + localhost_paths_to_remove = ["/localhost/BGP_NEIGHBOR/", + "/localhost/DEVICE_NEIGHBOR_METADATA/" + ] + localhost_keys_to_remove = [ + config_facts["BGP_NEIGHBOR"].keys() if config_facts.get("BGP_NEIGHBOR") else [], + config_facts["DEVICE_NEIGHBOR_METADATA"].keys() if config_facts.get("DEVICE_NEIGHBOR_METADATA") else [], + ] + if 'INTERFACE' in config_facts: + localhost_paths_to_remove.append("/localhost/INTERFACE/") + localhost_keys_to_remove.append(localhost_ip_prefix_interface_keys) + if 'PORTCHANNEL' in config_facts: + localhost_paths_to_remove.append("/localhost/PORTCHANNEL_MEMBER/") + localhost_paths_to_remove.append("/localhost/PORTCHANNEL_INTERFACE/") + localhost_keys_to_remove.append(localhost_pc_member_keys) + localhost_keys_to_remove.append(localhost_pc_interface_keys) + + for path, keys in zip(localhost_paths_to_remove, localhost_keys_to_remove): + for k in keys: + localhost_paths_list.append(path + k) + for path in localhost_paths_list: + json_patch_localhost.append({ + "op": "remove", + "path": path + }) + + ##################################### + # combine localhost and ASIC patch data + ##################################### + json_patch = json_patch_localhost + json_patch_asic + tmpfile = generate_tmpfile(duthost) + try: + logger.info("Applying patch (1/2) to remove cluster info (all except PORTCHANNEL, INTERFACE name).") + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + verify_bgp_peers_removed_from_asic(duthost, enum_rand_one_asic_namespace) + finally: + delete_tmpfile(duthost, tmpfile) + + # W/A TABLE:PORTCHANNEL, INTERFACE names needs to be removed in separate gcu apply operation + # https://github.com/sonic-net/sonic-buildimage/issues/24338 + json_patch_extra = [] + if 'PORTCHANNEL' in config_facts: + json_patch_extra = [ + { + "op": "remove", + "path": f"{json_namespace}/PORTCHANNEL" + } + ] + for key, _value in config_facts.get("PORTCHANNEL", {}).items(): + json_patch_extra.append({ + "op": "remove", + "path": "/localhost/PORTCHANNEL/{}".format(key), + }) + interface_paths_list = [] + interface_paths_to_remove = [f"{json_namespace}/INTERFACE/", "/localhost/INTERFACE/"] + interface_keys_to_remove = [asic_interface_keys, localhost_interface_keys] + for path, keys in zip(interface_paths_to_remove, interface_keys_to_remove): + for k in keys: + interface_paths_list.append(path + k) + for path in interface_paths_list: + json_patch_extra.append({ + "op": "remove", + "path": path + }) + + tmpfile_pc = generate_tmpfile(duthost) + try: + logger.info("Applying patch (2/2) to remove cluster info (PORTCHANNEL, INTERFACE name).") + output = apply_patch(duthost, json_data=json_patch_extra, dest_file=tmpfile_pc) + expect_op_success(duthost, output) + finally: + delete_tmpfile(duthost, tmpfile_pc) + + +def apply_patch_add_cluster(config_facts, + config_facts_localhost, + mg_facts, + duthost, + enum_rand_one_asic_namespace): + """ + Apply patch to add cluster information for a given ASIC namespace. + + Changes are perfomed to below tables: + + ACL_TABLE + BGP_NEIGHBOR + DEVICE_NEIGHBOR + DEVICE_NEIGHBOR_METADATA + PORTCHANNEL + PORTCHANNEL_INTERFACE + PORTCHANNEL_MEMBER + INTERFACE + BUFFER_PG + CABLE_LENGTH + PORT + PORT_QOS_MAP + """ + + logger.info("Adding cluster for namespace {} via apply-patch.".format(enum_rand_one_asic_namespace)) + + ###################### + # ASIC NAMESPACE + ###################### + json_patch_asic = [] + json_namespace = '' if enum_rand_one_asic_namespace is None else '/' + enum_rand_one_asic_namespace + pc_dict = {} + interface_dict = format_sonic_interface_dict(config_facts.get("INTERFACE", {})) + portchannel_interface_dict = format_sonic_interface_dict(config_facts.get("PORTCHANNEL_INTERFACE", {})) + portchannel_member_dict = format_sonic_interface_dict(config_facts.get("PORTCHANNEL_MEMBER", {}), + single_entry=False) + buffer_pg_dict = format_sonic_buffer_pg_dict(config_facts.get("BUFFER_PG", {})) + pc_dict = { + k: {ik: iv for ik, iv in v.items() if ik != "members"} + for k, v in config_facts.get("PORTCHANNEL", {}).items() + } + + # find active ports + active_interfaces = get_active_interfaces(config_facts) + + # PORTCHANNEL info needs to be added in separate gcu apply operation + # https://github.com/sonic-net/sonic-buildimage/issues/24338 + if pc_dict: + json_patch_pc = [ + { + "op": "add", + "path": f"{json_namespace}/PORTCHANNEL", + "value": pc_dict + } + ] + for pc_key, pc_value in pc_dict.items(): + json_patch_pc.append({ + "op": "add", + "path": "/localhost/PORTCHANNEL/{}".format(pc_key), + "value": pc_value + }) + tmpfile_pc = generate_tmpfile(duthost) + try: + logger.info("Applying patch (1/2) to add cluster info (PORTCHANNEL).") + output = apply_patch(duthost, json_data=json_patch_pc, dest_file=tmpfile_pc) + expect_op_success(duthost, output) + finally: + delete_tmpfile(duthost, tmpfile_pc) + + # op: add + json_patch_asic = [ + { + "op": "add", + "path": f"{json_namespace}/BGP_NEIGHBOR", + "value": config_facts["BGP_NEIGHBOR"] + }, + { + "op": "add", + "path": f"{json_namespace}/DEVICE_NEIGHBOR", + "value": config_facts["DEVICE_NEIGHBOR"] + }, + { + "op": "add", + "path": f"{json_namespace}/DEVICE_NEIGHBOR_METADATA", + "value": config_facts["DEVICE_NEIGHBOR_METADATA"] + }, + { + "op": "add", + "path": f"{json_namespace}/INTERFACE", + "value": interface_dict + }, + { + "op": "add", + "path": f"{json_namespace}/BUFFER_PG", + "value": buffer_pg_dict + }, + { + "op": "add", + "path": f"{json_namespace}/PORT_QOS_MAP", + "value": config_facts["PORT_QOS_MAP"] + } + ] + + if 'PORTCHANNEL' in config_facts: + json_patch_asic.append({ + "op": "add", + "path": f"{json_namespace}/PORTCHANNEL_MEMBER", + "value": portchannel_member_dict + }) + json_patch_asic.append({ + "op": "add", + "path": f"{json_namespace}/PORTCHANNEL_INTERFACE", + "value": portchannel_interface_dict + }) + + # table PORT changes + for interface in active_interfaces: + json_patch_asic.append({ + "op": "add", + "path": "{}/PORT/{}/admin_status".format(json_namespace, interface), + "value": "up" + }) + + # table CABLE_LENGTH changes + initial_cable_length_table = config_facts["CABLE_LENGTH"]["AZURE"] + cable_length_values = [int(v.rstrip("m")) for v in initial_cable_length_table.values()] + highest = max(cable_length_values) + for interface in active_interfaces: + json_patch_asic.append({ + "op": "add", + "path": "{}/CABLE_LENGTH/AZURE/{}".format(json_namespace, interface), + "value": f"{highest}m" + }) + + # table ACL_TABLE changes + json_patch_asic.append({ + "op": "add", + "path": f"{json_namespace}/ACL_TABLE/DATAACL", + "value": config_facts["ACL_TABLE"]["DATAACL"] + }) + json_patch_asic.append({ + "op": "add", + "path": f"{json_namespace}/ACL_TABLE/EVERFLOW", + "value": config_facts["ACL_TABLE"]["EVERFLOW"] + }) + json_patch_asic.append({ + "op": "add", + "path": f"{json_namespace}/ACL_TABLE/EVERFLOWV6", + "value": config_facts["ACL_TABLE"]["EVERFLOWV6"] + }) + + ###################### + # LOCALHOST NAMESPACE + ###################### + + json_patch_localhost = [] + + # INTERFACE keys: in localhost replace the interface name with the interface alias + localhost_interface_dict = {} + for key, value in interface_dict.items(): + if key.startswith('Ethernet-Rec'): + continue + parts = key.split('|') + updated_key = key + if len(parts) == 2: + port = parts[0] + alias = mg_facts['minigraph_port_name_to_alias_map'].get(port, port) + updated_key = "{}|{}".format(alias, parts[1]) + else: + updated_key = mg_facts['minigraph_port_name_to_alias_map'].get(key, key) + updated_key = updated_key.replace("/", "~1") + localhost_interface_dict[updated_key] = value + + # identify the keys to add + localhost_add_paths_list = [] + localhost_add_values_list = [] + for k, v in list(config_facts["BGP_NEIGHBOR"].items()): + localhost_add_paths_list.append('/localhost/BGP_NEIGHBOR/{}'.format(k)) + localhost_add_values_list.append(v) + for k, v in list(config_facts["DEVICE_NEIGHBOR"].items()): + localhost_add_paths_list.append('/localhost/DEVICE_NEIGHBOR/{}'.format(k)) + localhost_add_values_list.append(v) + for k, v in list(config_facts["DEVICE_NEIGHBOR_METADATA"].items()): + localhost_add_paths_list.append('/localhost/DEVICE_NEIGHBOR_METADATA/{}'.format(k)) + localhost_add_values_list.append(v) + for k, v in list(localhost_interface_dict.items()): + localhost_add_paths_list.append("/localhost/INTERFACE/{}".format(k)) + localhost_add_values_list.append(v) + + if 'PORTCHANNEL' in config_facts: + # PORTCHANNEL INTERFACE + localhost_pc_interface_dict = {} + for key, value in portchannel_interface_dict.items(): + updated_key = key.replace('/', '~1') + localhost_pc_interface_dict[updated_key] = value + # PORTCHANNEL_MEMBER keys + localhost_pc_member_dict = {} + for key, value in portchannel_member_dict.items(): + parts = key.split('|') + updated_key = key + if len(parts) == 2: + port = parts[1] + alias = mg_facts['minigraph_port_name_to_alias_map'].get(port, port) + updated_key = "{}|{}".format(parts[0], alias) + updated_key = updated_key.replace("/", "~1") + localhost_pc_member_dict[updated_key] = value + # for k, v in list(pc_dict.items()): + # localhost_add_paths_list.append("/localhost/PORTCHANNEL/{}".format(k)) + # localhost_add_values_list.append(v) + for k, v in list(localhost_pc_interface_dict.items()): + localhost_add_paths_list.append("/localhost/PORTCHANNEL_INTERFACE/{}".format(k)) + localhost_add_values_list.append(v) + for k, v in list(localhost_pc_member_dict.items()): + localhost_add_paths_list.append("/localhost/PORTCHANNEL_MEMBER/{}".format(k)) + localhost_add_values_list.append(v) + + for path, value in zip(localhost_add_paths_list, localhost_add_values_list): + json_patch_localhost.append({ + "op": "add", + "path": path, + "value": value + }) + + json_patch_localhost.append({ + "op": "add", + "path": "/localhost/ACL_TABLE/DATAACL/ports", + "value": config_facts_localhost["ACL_TABLE"]["DATAACL"]["ports"] + }) + json_patch_localhost.append({ + "op": "add", + "path": "/localhost/ACL_TABLE/EVERFLOW/ports", + "value": config_facts_localhost["ACL_TABLE"]["EVERFLOW"]["ports"] + }) + json_patch_localhost.append({ + "op": "add", + "path": "/localhost/ACL_TABLE/EVERFLOWV6/ports", + "value": config_facts_localhost["ACL_TABLE"]["EVERFLOWV6"]["ports"] + }) + + ##################################### + # combine localhost and ASIC patch data + ##################################### + json_patch = json_patch_localhost + json_patch_asic + tmpfile = generate_tmpfile(duthost) + try: + logger.info("Applying patch (2/2) to add cluster info (all except PORTCHANNEL).") + output = apply_patch(duthost, json_data=json_patch, dest_file=tmpfile) + expect_op_success(duthost, output) + finally: + delete_tmpfile(duthost, tmpfile) + + +def format_sonic_interface_dict(interface_dict, single_entry=True): + """ + Converts a SONiC interface dictionary into the correct format so the formatted value can be used + as the 'value' in a JSON patch. + + - Ensures interfaces exist as standalone keys. + - Converts IP addresses into the "Interface|IP" format. + """ + formatted_interface_dict = {} + + for key, values in interface_dict.items(): + if isinstance(values, dict): # if IPs are defined under the interface + if single_entry: + formatted_interface_dict[key] = {} + for ip in values.keys(): + formatted_interface_dict[f"{key}|{ip}"] = {} + else: + if single_entry: + formatted_interface_dict[key] = {} + + return formatted_interface_dict + + +def format_sonic_buffer_pg_dict(buffer_pg_dict): + """ + Converts a SONiC interface dictionary into the correct format so the formatted value can be used + as the 'value' in a JSON patch. + """ + formatted_dict = {} + for key, values in buffer_pg_dict.items(): + if isinstance(values, dict): + for pg_num_key, value in values.items(): + formatted_dict[f"{key}|{pg_num_key}"] = value + return formatted_dict + + +# ----------------------------- +# Setup Fixtures/functions +# ----------------------------- + +@pytest.fixture(scope="module", params=[False, True]) +def acl_config_scenario(request): + return request.param + + +# Setting to false due to kvm data traffic issue failing the test case. Need to be enabled after investigation. +# Issue: https://github.com/sonic-net/sonic-mgmt/issues/21775 +@pytest.fixture(scope="module", params=[False]) +def data_traffic_scenario(request): + return request.param + + +def setup_acl_config(duthost, ip_netns_namespace_prefix): + logger.info("Adding acl config.") + remove_dataacl_table_single_dut("DATAACL", duthost) + duthost.command("{} config acl add table {} {} -s {}".format( + ip_netns_namespace_prefix, ACL_TABLE_NAME, ACL_TABLE_TYPE_L3, ACL_TABLE_STAGE_EGRESS)) + duthost.copy(src=ACL_RULE_FILE_PATH, dest=ACL_RULE_DST_FILE) + duthost.shell("{} acl-loader update full --table_name {} {}".format( + ip_netns_namespace_prefix, ACL_TABLE_NAME, ACL_RULE_DST_FILE)) + acl_tables = duthost.command("{} show acl table".format(ip_netns_namespace_prefix))["stdout_lines"] + acl_rules = duthost.command("{} show acl rule".format(ip_netns_namespace_prefix))["stdout_lines"] + logging.info(('\n'.join(acl_tables))) + logging.info(('\n'.join(acl_rules))) + + +def remove_acl_config(duthost, ip_netns_namespace_prefix): + logger.info("Removing acl config.") + config_reload(duthost, config_source="minigraph", safe_reload=True) + acl_tables = duthost.command("{} show acl table".format(ip_netns_namespace_prefix))["stdout_lines"] + acl_rules = duthost.command("{} show acl rule".format(ip_netns_namespace_prefix))["stdout_lines"] + logging.info(('\n'.join(acl_tables))) + logging.info(('\n'.join(acl_rules))) + + +@pytest.fixture(scope="module") +def setup_static_route(tbinfo, duthosts, enum_downstream_dut_hostname, + enum_rand_one_frontend_asic_index, + rand_bgp_neigh_ip_name): + duthost = duthosts[enum_downstream_dut_hostname] + bgp_neigh_ip, bgp_neigh_name = rand_bgp_neigh_ip_name + logger.info("Adding static route {} to be routed via bgp neigh {}.".format(STATIC_DST_IP, bgp_neigh_ip)) + exabgp_port = get_exabgp_port_for_neighbor(tbinfo, bgp_neigh_name, EXABGP_BASE_PORT) + route_exists = verify_routev4_existence(duthost, enum_rand_one_frontend_asic_index, + STATIC_DST_IP, should_exist=True) + if route_exists: + logger.warning("Route exists already - will try to clear") + clear_static_route(tbinfo, duthost, STATIC_DST_IP) + add_static_route(tbinfo, bgp_neigh_ip, exabgp_port, ip=STATIC_DST_IP, nhipv4=NHIPV4) + wait_until(10, 1, 0, verify_routev4_existence, duthost, + enum_rand_one_frontend_asic_index, STATIC_DST_IP, should_exist=True) + + yield + + logger.info("Removing static route {} .".format(STATIC_DST_IP)) + remove_static_route(tbinfo, bgp_neigh_ip, exabgp_port, ip=STATIC_DST_IP, nhipv4=NHIPV4) + wait_until(10, 1, 0, verify_routev4_existence, duthost, + enum_rand_one_frontend_asic_index, STATIC_DST_IP, should_exist=False) + + +@pytest.fixture(scope="function") +def initialize_random_variables(enum_downstream_dut_hostname, + enum_upstream_dut_hostname, + enum_rand_one_frontend_asic_index, + enum_rand_one_asic_namespace, + ip_netns_namespace_prefix, + cli_namespace_prefix, + rand_bgp_neigh_ip_name): + return enum_downstream_dut_hostname, enum_upstream_dut_hostname, enum_rand_one_frontend_asic_index, \ + enum_rand_one_asic_namespace, ip_netns_namespace_prefix, cli_namespace_prefix, rand_bgp_neigh_ip_name + + +@pytest.fixture(scope="function") +def initialize_facts(mg_facts, + config_facts, + config_facts_localhost): + return mg_facts, config_facts, config_facts_localhost + + +@pytest.fixture(scope="function") +def setup_add_cluster(tbinfo, + duthosts, + localhost, + initialize_random_variables, + initialize_facts, + ptfadapter, + loganalyzer, + acl_config_scenario, + setup_static_route, + data_traffic_scenario): + """ + This setup fixture prepares the Downstream LC by applying a patch to remove + and then re-add the cluster configuration. + + The purpose is to prepare the DUT host for test cases that validate functionality + after adding a cluster via apply-patch. + The fixture reads the running configuration and constructs patches to remove + the current config from a running namespace. + After verifying successful removal, it re-adds the configuration and validates that it was successfully restored. + + **Setup steps - applied to the Downstream LC:** + 1. Save the original configuration. + 2. Remove the cluster from a randomly selected namespace. + 3. Verify BGP information, route table, and interface details to ensure everything has been removed as expected. + 4. Perform data verification in the upstream → downlink direction, targeting a static route, which should now fail. + 5. Save the configuration and reboot the system so that it initializes clear from cluster information + 6. Re-add the cluster to the randomly selected namespace. + 7. Verify BGP information, route table, and interface details to ensure everything is restored as expected. + 8. Add ACL configuration based on the test parameter value. + + **Teardown steps:** + The setup logic already re-applies the initial cluster configuration for the namespace. + The only recovery needed during teardown is for the ACL configuration: + 1. Restore the ACL configuration to its initial values. + """ + + # initial test env + enum_downstream_dut_hostname, enum_upstream_dut_hostname, enum_rand_one_frontend_asic_index, \ + enum_rand_one_asic_namespace, ip_netns_namespace_prefix, cli_namespace_prefix, \ + rand_bgp_neigh_ip_name = initialize_random_variables + mg_facts, config_facts, config_facts_localhost = initialize_facts + duthost = duthosts[enum_downstream_dut_hostname] + # Check if the device is a modular chassis and the topology is T2 + is_chassis = duthost.get_facts().get("modular_chassis") + if not (is_chassis and tbinfo['topo']['type'] == 't2' and duthost.facts['switch_type'] == "voq"): + # Skip the test if the setup is not T2 Chassis + pytest.skip("Test is Applicable for T2 VOQ Chassis Setup") + duthost_src = duthosts[enum_upstream_dut_hostname] + asic_id = enum_rand_one_frontend_asic_index + asic_id_src = None + all_asic_ids = duthost_src.get_asic_ids() + for asic in all_asic_ids: + if duthost_src == duthost and asic == asic_id: + continue + asic_id_src = asic + break + bgp_neigh_ip, _bgp_neigh_name = rand_bgp_neigh_ip_name + pytest_assert( + asic_id_src is not None, "Couldn't find an asic id to be used for sending traffic. \ + Reserved asic id: {}. All available asic ids: {}".format( + asic_id, all_asic_ids + ) + ) + initial_buffer_pg_info = get_cfg_info_from_dut(duthost, 'BUFFER_PG', enum_rand_one_asic_namespace) + with allure.step("Verification before removing cluster"): + for host_device in duthosts: + if host_device.is_supervisor_node(): + continue + logger.info(host_device.shell('show ip bgp summary -d all')) + logger.info(host_device.shell('show ipv6 bgp summary -d all')) + route_exists = verify_routev4_existence(duthost, asic_id, STATIC_DST_IP, should_exist=True) + route_exists_src = verify_routev4_existence(duthost_src, asic_id_src, STATIC_DST_IP, should_exist=True) + pytest_assert(route_exists, "Static route {} doesn't exist on downstream DUT before cluster removal." + .format(STATIC_DST_IP)) + pytest_assert(route_exists_src, "Static route {} doesn't exist on upstream DUT before cluster removal." + .format(STATIC_DST_IP)) + if data_traffic_scenario: + logger.info("Sending traffic from upstream DUT to downstream DUT before cluster removal.") + send_and_verify_traffic(tbinfo, duthost_src, duthost, asic_id_src, asic_id, + ptfadapter, dst_ip=STATIC_DST_IP, count=10, expect_error=False) + + with allure.step("Removing cluster info for namespace"): + # disable loganalyzer during cluster removal + logger.info("Disabling loganalyzer before starting cluster removal.") + if loganalyzer and loganalyzer[duthost.hostname]: + loganalyzer[duthost.hostname].add_start_ignore_mark() + + if len(config_facts["BUFFER_PG"]) <= 6: # num of active interfaces = num of pg lossless profiles + logger.info("Removal method gcu - min setup.") + apply_patch_remove_cluster(config_facts, + config_facts_localhost, + mg_facts, + duthost, + enum_rand_one_asic_namespace, + cli_namespace_prefix) + else: + logger.info("Removal method sonic-db-cli - mid-max setup.") + remove_cluster_via_sonic_db_cli(config_facts, + config_facts_localhost, + mg_facts, + duthost, + enum_rand_one_asic_namespace, + cli_namespace_prefix) + + # Verify routes removed + wait_until(5, 1, 0, verify_routev4_existence, duthost, + enum_rand_one_frontend_asic_index, bgp_neigh_ip, should_exist=False) + wait_until(5, 1, 0, verify_routev4_existence, duthost, + enum_rand_one_frontend_asic_index, STATIC_DST_IP, should_exist=False) + + # re-enabling loganalyzer during cluster removal + logger.info("Re-enabling loganalyzer after cluster removal.") + if loganalyzer and loganalyzer[duthost.hostname]: + loganalyzer[duthost.hostname].add_end_ignore_mark() + + with allure.step("Reload the system with config reload"): + duthost.shell("config save -y") + config_reload(duthost, config_source='config_db', safe_reload=True) + pytest_assert(wait_until(300, 20, 0, duthost.critical_services_fully_started), + "All critical services should be fully started!") + pytest_assert(wait_until(1200, 20, 0, check_interface_status_of_up_ports, duthost), + "Not all ports that are admin up on are operationally up") + + with allure.step("Verify config after reload"): + tmpfile = generate_tmpfile(duthost) + output = apply_patch(duthost, json_data=[], dest_file=tmpfile) + expect_op_success(duthost, output) + + with allure.step("Adding cluster info for namespace"): + apply_patch_add_cluster(config_facts, + config_facts_localhost, + mg_facts, + duthost, + enum_rand_one_asic_namespace) + # Verify routes added + wait_until(5, 1, 0, verify_routev4_existence, + duthost, enum_rand_one_frontend_asic_index, bgp_neigh_ip, should_exist=True) + wait_until(5, 1, 0, verify_routev4_existence, + duthost, enum_rand_one_frontend_asic_index, STATIC_DST_IP, should_exist=True) + # Verify buffer pg + buffer_pg_info_add_interfaces = get_cfg_info_from_dut(duthost, 'BUFFER_PG', enum_rand_one_asic_namespace) + pytest_assert(buffer_pg_info_add_interfaces == initial_buffer_pg_info, + "Didn't find expected BUFFER_PG info in CONFIG_DB after adding back the interfaces.") + + if acl_config_scenario: + setup_acl_config(duthost, ip_netns_namespace_prefix) + + yield + + if acl_config_scenario: + remove_acl_config(duthost, ip_netns_namespace_prefix) + + +# ----------------------------- +# Test Definitions +# ----------------------------- + +def test_add_cluster(tbinfo, + duthosts, + initialize_random_variables, + ptfadapter, + loganalyzer, + acl_config_scenario, + cli_namespace_prefix, + setup_add_cluster, + data_traffic_scenario): + """ + Validates the functionality of the Downstream Linecard after adding a cluster. + + Performs lossless data traffic scenarios for both ACL and non-ACL cases. + Verifies successful data transmission, queue counters, and ACL rule match counters. + """ + + # initial test env + enum_downstream_dut_hostname, enum_upstream_dut_hostname, enum_rand_one_frontend_asic_index, \ + enum_rand_one_asic_namespace, ip_netns_namespace_prefix, cli_namespace_prefix, \ + rand_bgp_neigh_ip_name = initialize_random_variables + duthost = duthosts[enum_downstream_dut_hostname] + duthost_up = duthosts[enum_upstream_dut_hostname] + asic_id = enum_rand_one_frontend_asic_index + asic_id_src = None + asic_id_src_up = None + for asic in duthost.get_asic_ids(): + if asic == asic_id: + continue + asic_id_src = asic + break + for asic in duthost_up.get_asic_ids(): + asic_id_src_up = asic + break + + pytest_assert( + asic_id_src is not None, "Couldn't find an asic id to be used for sending traffic. \ + Reserved asic id: {}. All available asic ids: {}".format( + asic_id, duthost.get_asic_ids() + ) + ) + pytest_assert( + asic_id_src is not None, "Couldn't find an asic id to be used for sending traffic from upstream. \ + All available asic ids: {}".format( + duthost_up.get_asic_ids() + ) + ) + + if data_traffic_scenario: + # Traffic scenarios applied in non-acl, acl scenario + traffic_scenarios = [ + {"direction": "upstream->downstream", "dst_ip": STATIC_DST_IP, "count": 1000, "dscp": 3, + "sport": 1234, "dport": 50, "verify": True, "expect_error": False}, + {"direction": "downstream->downstream", "dst_ip": STATIC_DST_IP, "count": 1000, "dscp": 3, + "sport": 1234, "dport": 50, "verify": True, "expect_error": False} + ] + if acl_config_scenario: + traffic_scenarios = [ + {"direction": "upstream->downstream", "dst_ip": STATIC_DST_IP, "count": 1000, "dscp": 3, + "sport": 5000, "dport": 50, "verify": True, "expect_error": False, "match_rule": "RULE_100"}, + {"direction": "upstream->downstream", "dst_ip": STATIC_DST_IP, "count": 1000, "dscp": 3, + "sport": 1234, "dport": 8080, "verify": True, "expect_error": True, "match_rule": "RULE_200"}, + {"direction": "upstream->downstream", "dst_ip": STATIC_DST_IP, "count": 1000, "dscp": 3, + "sport": 1234, "dport": 50, "verify": True, "expect_error": False, "match_rule": None}, + {"direction": "downstream->downstream", "dst_ip": STATIC_DST_IP, "count": 1000, "dscp": 3, + "sport": 5000, "dport": 50, "verify": True, "expect_error": False, "match_rule": "RULE_100"}, + {"direction": "downstream->downstream", "dst_ip": STATIC_DST_IP, "count": 1000, "dscp": 3, + "sport": 1234, "dport": 8080, "verify": True, "expect_error": True, "match_rule": "RULE_200"}, + {"direction": "downstream->downstream", "dst_ip": STATIC_DST_IP, "count": 1000, "dscp": 3, + "sport": 1234, "dport": 50, "verify": True, "expect_error": False, "match_rule": None} + ] + + for traffic_scenario in traffic_scenarios: + logger.info("Starting Data Traffic Scenario: {}".format(traffic_scenario)) + if traffic_scenario["direction"] == "upstream->downstream": + src_duthost = duthost_up + src_asic_index = asic_id_src_up + elif traffic_scenario["direction"] == "downstream->downstream": + src_duthost = duthost + src_asic_index = asic_id_src + else: + pytest_assert("Unsupported direction for traffic scenario {}.".format(traffic_scenario["direction"])) + + if acl_config_scenario: + duthost.shell('{} aclshow -c'.format(ip_netns_namespace_prefix)) + + send_and_verify_traffic(tbinfo, src_duthost, duthost, src_asic_index, asic_id, + ptfadapter, + dst_ip=traffic_scenario["dst_ip"], + dscp=traffic_scenario["dscp"], + count=traffic_scenario["count"], + sport=traffic_scenario["sport"], + dport=traffic_scenario["dport"], + verify=traffic_scenario["verify"], + expect_error=traffic_scenario["expect_error"]) + + if acl_config_scenario: + acl_counters = duthost.show_and_parse('{} aclshow -a'.format(ip_netns_namespace_prefix)) + for acl_counter in acl_counters: + if acl_counter["rule name"] in ACL_RULE_SKIP_VERIFICATION_LIST: + continue + pytest_assert(acl_counter["packets count"] == str(traffic_scenario["count"]) + if acl_counter["rule name"] == traffic_scenario["match_rule"] + else acl_counter["packets count"] == '0', + "Acl rule {} statistics are not as expected. Found value {}" + .format(acl_counter["rule name"], acl_counter["packets count"]))