diff --git a/tests/common/dualtor/dual_tor_utils.py b/tests/common/dualtor/dual_tor_utils.py index 31366048e1e..1abc4ca34cf 100644 --- a/tests/common/dualtor/dual_tor_utils.py +++ b/tests/common/dualtor/dual_tor_utils.py @@ -921,6 +921,9 @@ def count_matched_packets_all_ports(ptfadapter, exp_packet, exp_tunnel_pkt, port return port_packet_count +# behavior has changed with such that ecmp groups that span across multiple +# mux interfaces are not balanced. Instead we expect packets to be sent to +# a single mux interface. def check_nexthops_balance(rand_selected_dut, ptfadapter, dst_server_addr, @@ -993,6 +996,69 @@ def check_nexthops_balance(rand_selected_dut, pc)) +# verify nexthops are only sent to single active or standby mux +def check_nexthops_single_downlink(rand_selected_dut, + ptfadapter, + dst_server_addr, + tbinfo, + downlink_ints): + HASH_KEYS = ["src-port", "dst-port", "src-ip"] + # expect this packet to be sent to downlinks (active mux) and uplink (stanby mux) + expected_downlink_ports = [get_ptf_server_intf_index(rand_selected_dut, tbinfo, iface) for iface in downlink_ints] + expected_uplink_ports = list() + expected_uplink_portchannels = list() + portchannel_ports = get_t1_ptf_pc_ports(rand_selected_dut, tbinfo) + for pc, intfs in portchannel_ports.items(): + expected_uplink_portchannels.append(pc) + for member in intfs: + expected_uplink_ports.append(int(member.strip("eth"))) + logging.info("Expecting packets in downlink ports {}".format(expected_downlink_ports)) + logging.info("Expecting packets in uplink ports {}".format(expected_uplink_ports)) + + ptf_t1_intf = random.choice(get_t1_ptf_ports(rand_selected_dut, tbinfo)) + port_packet_count = dict() + packets_to_send = generate_hashed_packet_to_server(ptfadapter, rand_selected_dut, HASH_KEYS, dst_server_addr, 10000) + for send_packet, exp_pkt, exp_tunnel_pkt in packets_to_send: + testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), send_packet, count=1) + # expect multi-mux nexthops to focus packets to one downlink + all_allowed_ports = expected_downlink_ports + ptf_port_count = count_matched_packets_all_ports(ptfadapter, + exp_packet=exp_pkt, + exp_tunnel_pkt=exp_tunnel_pkt, + ports=all_allowed_ports, + timeout=0.1, + count=1) + + for ptf_idx, pkt_count in ptf_port_count.items(): + port_packet_count[ptf_idx] = port_packet_count.get(ptf_idx, 0) + pkt_count + + logging.info("Received packets in ports: {}".format(str(port_packet_count))) + expect_packet_num = 10000 + for downlink_int in expected_downlink_ports: + # packets should be either 0 or expect_packet_num: + count = port_packet_count.get(downlink_int, 0) + logging.info("Packets received on downlink port {}: {}".format(downlink_int, count)) + if count > 0 and count != expect_packet_num: + all_pkts = False + pt_assert(all_pkts, "Packets not sent down single active port {}".format(downlink_int)) + + if len(downlink_ints) == 0: + # All nexthops are now connected to standby mux, and the packets will be sent towards a single portchanel int + # Check if uplink distribution is towards a single portchannel + for pc, intfs in portchannel_ports.items(): + count = 0 + # Collect the packets count within a single portchannel + for member in intfs: + uplink_int = int(member.strip("eth")) + count = count + port_packet_count.get(uplink_int, 0) + logging.info("Packets received on portchannel {}: {}".format(pc, count)) + + if count > 0 and count != expect_packet_num: + all_pkts = False + pt_assert(all_pkts, "Packets not sent up single standby port {}".format( + pc)) + + def verify_upstream_traffic(host, ptfadapter, tbinfo, itfs, server_ip, pkt_num = 100, drop = False): """ @summary: Helper function for verifying upstream packets diff --git a/tests/dualtor/test_orchagent_active_tor_downstream.py b/tests/dualtor/test_orchagent_active_tor_downstream.py index 6979af2bd07..62f3a95c00b 100644 --- a/tests/dualtor/test_orchagent_active_tor_downstream.py +++ b/tests/dualtor/test_orchagent_active_tor_downstream.py @@ -12,7 +12,7 @@ from tests.common.dualtor.dual_tor_utils import crm_neighbor_checker from tests.common.dualtor.dual_tor_utils import build_packet_to_server from tests.common.dualtor.dual_tor_utils import get_interface_server_map -from tests.common.dualtor.dual_tor_utils import check_nexthops_balance +from tests.common.dualtor.dual_tor_utils import check_nexthops_single_downlink from tests.common.dualtor.dual_tor_utils import add_nexthop_routes, remove_static_routes from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports from tests.common.dualtor.server_traffic_utils import ServerTrafficMonitor @@ -160,29 +160,26 @@ def test_downstream_ecmp_nexthops( add_nexthop_routes(rand_selected_dut, dst_server_addr, nexthops=nexthop_servers) try: - logging.info("Verify traffic to this route destination is distributed to four server ports") - check_nexthops_balance(rand_selected_dut, ptfadapter, dst_server_addr, tbinfo, - nexthop_interfaces, nexthops_count) + logging.info("Verify traffic to this route destination is sent to single downlink or uplink") + check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr, + tbinfo, nexthop_interfaces) ### Sequentially set four mux states to standby for index, interface in enumerate(nexthop_interfaces): uplink_ports_active = index + 1 logging.info("Simulate {} mux state change to Standby".format(nexthop_servers[index])) set_mux_state(rand_selected_dut, tbinfo, 'standby', [interface], toggle_all_simulator_ports) - logging.info("Verify traffic to this route destination is distributed to"\ - " {} server ports and {} tunnel nexthop".format( - nexthops_count-uplink_ports_active, uplink_ports_active)) - check_nexthops_balance(rand_selected_dut, ptfadapter, dst_server_addr, tbinfo, - nexthop_interfaces[uplink_ports_active:nexthops_count], nexthops_count) + logging.info("Verify traffic to this route destination is sent to single downlink or uplink") + check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr, + tbinfo, nexthop_interfaces) ### Revert two mux states to active for index, interface in reversed(list(enumerate(nexthop_interfaces))): logging.info("Simulate {} mux state change back to Active".format(nexthop_servers[index])) set_mux_state(rand_selected_dut, tbinfo, 'active', [interface], toggle_all_simulator_ports) - logging.info("Verify traffic to this route destination is distributed to"\ - " {} server ports and {} tunnel nexthop".format(nexthops_count-index, index)) - check_nexthops_balance(rand_selected_dut, ptfadapter, dst_server_addr, tbinfo, - nexthop_interfaces[index:nexthops_count], nexthops_count) + logging.info("Verify traffic to this route destination is sent to single downlink or uplink") + check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr, + tbinfo, nexthop_interfaces) finally: ### Remove the nexthop route remove_static_routes(rand_selected_dut, dst_server_addr)