[202012][dualtor] fixing test_downstream_ecmp_nexthops

tests/common/dualtor/dual_tor_utils.py

@@ -921,6 +921,9 @@ def count_matched_packets_all_ports(ptfadapter, exp_packet, exp_tunnel_pkt, port
     return port_packet_count
 
 
+# behavior has changed with such that ecmp groups that span across multiple
+# mux interfaces are not balanced. Instead we expect packets to be sent to
+# a single mux interface.
 def check_nexthops_balance(rand_selected_dut,
     ptfadapter,
     dst_server_addr,
@@ -993,6 +996,69 @@ def check_nexthops_balance(rand_selected_dut,
                     pc))
 
 
+# verify nexthops are only sent to single active or standby mux
+def check_nexthops_single_downlink(rand_selected_dut,
+    ptfadapter,
+    dst_server_addr,
+    tbinfo,
+    downlink_ints):
+    HASH_KEYS = ["src-port", "dst-port", "src-ip"]
+    # expect this packet to be sent to downlinks (active mux) and uplink (stanby mux)
+    expected_downlink_ports =  [get_ptf_server_intf_index(rand_selected_dut, tbinfo, iface) for iface in downlink_ints]
+    expected_uplink_ports = list()
+    expected_uplink_portchannels = list()
+    portchannel_ports = get_t1_ptf_pc_ports(rand_selected_dut, tbinfo)
+    for pc, intfs in portchannel_ports.items():
+        expected_uplink_portchannels.append(pc)
+        for member in intfs:
+            expected_uplink_ports.append(int(member.strip("eth")))
+    logging.info("Expecting packets in downlink ports {}".format(expected_downlink_ports))
+    logging.info("Expecting packets in uplink ports {}".format(expected_uplink_ports))
+
+    ptf_t1_intf = random.choice(get_t1_ptf_ports(rand_selected_dut, tbinfo))
+    port_packet_count = dict()
+    packets_to_send = generate_hashed_packet_to_server(ptfadapter, rand_selected_dut, HASH_KEYS, dst_server_addr, 10000)
+    for send_packet, exp_pkt, exp_tunnel_pkt in packets_to_send:
+        testutils.send(ptfadapter, int(ptf_t1_intf.strip("eth")), send_packet, count=1)
+        # expect multi-mux nexthops to focus packets to one downlink
+        all_allowed_ports = expected_downlink_ports
+        ptf_port_count = count_matched_packets_all_ports(ptfadapter,
+                                            exp_packet=exp_pkt,
+                                            exp_tunnel_pkt=exp_tunnel_pkt,
+                                            ports=all_allowed_ports,
+                                            timeout=0.1,
+                                            count=1)
+
+        for ptf_idx, pkt_count in ptf_port_count.items():
+            port_packet_count[ptf_idx] = port_packet_count.get(ptf_idx, 0) + pkt_count
+
+    logging.info("Received packets in ports: {}".format(str(port_packet_count)))
+    expect_packet_num = 10000
+    for downlink_int in expected_downlink_ports:
+        # packets should be either 0 or expect_packet_num:
+        count = port_packet_count.get(downlink_int, 0)
+        logging.info("Packets received on downlink port {}: {}".format(downlink_int, count))
+        if count > 0 and count != expect_packet_num:
+            all_pkts = False
+            pt_assert(all_pkts, "Packets not sent down single active port {}".format(downlink_int))
+
+    if len(downlink_ints) == 0:
+        # All nexthops are now connected to standby mux, and the packets will be sent towards a single portchanel int
+        # Check if uplink distribution is towards a single portchannel
+        for pc, intfs in portchannel_ports.items():
+            count = 0
+            # Collect the packets count within a single portchannel
+            for member in intfs:
+                uplink_int = int(member.strip("eth"))
+                count = count + port_packet_count.get(uplink_int, 0)
+            logging.info("Packets received on portchannel {}: {}".format(pc, count))
+
+            if count > 0 and count != expect_packet_num:
+                all_pkts = False
+                pt_assert(all_pkts, "Packets not sent up single standby port {}".format(
+                    pc))
+
+
 def verify_upstream_traffic(host, ptfadapter, tbinfo, itfs, server_ip, pkt_num = 100, drop = False):
     """
     @summary: Helper function for verifying upstream packets

tests/dualtor/test_orchagent_active_tor_downstream.py

@@ -12,7 +12,7 @@
 from tests.common.dualtor.dual_tor_utils import crm_neighbor_checker
 from tests.common.dualtor.dual_tor_utils import build_packet_to_server
 from tests.common.dualtor.dual_tor_utils import get_interface_server_map
-from tests.common.dualtor.dual_tor_utils import check_nexthops_balance
+from tests.common.dualtor.dual_tor_utils import check_nexthops_single_downlink
 from tests.common.dualtor.dual_tor_utils import add_nexthop_routes, remove_static_routes
 from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports
 from tests.common.dualtor.server_traffic_utils import ServerTrafficMonitor
@@ -160,29 +160,26 @@ def test_downstream_ecmp_nexthops(
     add_nexthop_routes(rand_selected_dut, dst_server_addr, nexthops=nexthop_servers)
 
     try:
-        logging.info("Verify traffic to this route destination is distributed to four server ports")
-        check_nexthops_balance(rand_selected_dut, ptfadapter, dst_server_addr, tbinfo,
-            nexthop_interfaces, nexthops_count)
+        logging.info("Verify traffic to this route destination is sent to single downlink or uplink")
+        check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr,
+                                       tbinfo, nexthop_interfaces)
 
         ### Sequentially set four mux states to standby
         for index, interface in enumerate(nexthop_interfaces):
             uplink_ports_active = index + 1
             logging.info("Simulate {} mux state change to Standby".format(nexthop_servers[index]))
             set_mux_state(rand_selected_dut, tbinfo, 'standby', [interface], toggle_all_simulator_ports)
-            logging.info("Verify traffic to this route destination is distributed to"\
-                " {} server ports and {} tunnel nexthop".format(
-                    nexthops_count-uplink_ports_active, uplink_ports_active))
-            check_nexthops_balance(rand_selected_dut, ptfadapter, dst_server_addr, tbinfo,
-                nexthop_interfaces[uplink_ports_active:nexthops_count], nexthops_count)
+            logging.info("Verify traffic to this route destination is sent to single downlink or uplink")
+            check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr,
+                                       tbinfo, nexthop_interfaces)
 
         ### Revert two mux states to active
         for index, interface in reversed(list(enumerate(nexthop_interfaces))):
             logging.info("Simulate {} mux state change back to Active".format(nexthop_servers[index]))
             set_mux_state(rand_selected_dut, tbinfo, 'active', [interface], toggle_all_simulator_ports)
-            logging.info("Verify traffic to this route destination is distributed to"\
-                " {} server ports and {} tunnel nexthop".format(nexthops_count-index, index))
-            check_nexthops_balance(rand_selected_dut, ptfadapter, dst_server_addr, tbinfo,
-                nexthop_interfaces[index:nexthops_count], nexthops_count)
+            logging.info("Verify traffic to this route destination is sent to single downlink or uplink")
+            check_nexthops_single_downlink(rand_selected_dut, ptfadapter, dst_server_addr,
+                                       tbinfo, nexthop_interfaces)
     finally:
         ### Remove the nexthop route
         remove_static_routes(rand_selected_dut, dst_server_addr)