[test_fgnhg] Add test to validate FG ECMP packet flow when neighbors are not resolved

ansible/roles/test/files/ptftests/fg_ecmp_test.py

@@ -28,6 +28,64 @@
 from ptf.testutils import *
 
 PERSIST_MAP = '/tmp/fg_ecmp_persist_map.json'
+MAX_ONE_PERCENT_LOSS = 0.01
+
+def verify_packet_warm(test, pkt, port, device_number=0, timeout=None, n_timeout=None):
+    # This packet verification function accounts for possible flood during warm boot
+    # We ensure that packets are received on the expected port, and return a special
+    # return value of -1 to denote that a flood had occured. The caller can use the 
+    # special return value to identify how many packets were flooded. 
+
+    if timeout is None:
+        timeout = ptf.ptfutils.default_timeout
+    if n_timeout is None:
+        n_timeout = ptf.ptfutils.default_negative_timeout
+    logging.debug("Checking for pkt on device %d, port %r", device_number, port)
+    result = dp_poll(test, device_number=device_number, timeout=timeout, exp_pkt=pkt)
+    verify_no_other_packets(test, device_number=device_number, timeout=n_timeout)
+
+    if isinstance(result, test.dataplane.PollSuccess):
+        if result.port != port:
+            # Flood case, check if packet rcvd on expected port as well
+            verify_packet(test, pkt, port)
+            return (-1, None)
+        else:
+            return (port, result.packet)
+
+    assert(isinstance(result, test.dataplane.PollFailure))
+    test.fail("Did not receive expected packet on any of ports %r for device %d.\n%s"
+                % (ports, device_number, result.format()))
+    return (0, None)
+
+def verify_packet_any_port_lossy(test, pkt, ports=[], device_number=0, timeout=None, n_timeout=None):
+    # This packet verification function accounts for possible loss of packet due to route table change
+    # We ensure that packets are received on the expected ports, and return a special
+    # return value of -1 to denote that a packet loss occured. The caller can use the 
+    # special return value to identify how many packets were lost and check if loss is within acceptable range
+
+    if timeout is None:
+        timeout = ptf.ptfutils.default_timeout
+    if n_timeout is None:
+        n_timeout = ptf.ptfutils.default_negative_timeout
+    logging.debug("Checking for pkt on device %d, port %r", device_number, ports)
+    result = dp_poll(test, device_number=device_number, timeout=timeout, exp_pkt=pkt)
+    verify_no_other_packets(test, device_number=device_number, timeout=n_timeout)
+
+    if isinstance(result, test.dataplane.PollSuccess):
+        if result.port in ports:
+            return (ports.index(result.port), result.packet)
+        else:
+            test.fail(
+                "Received expected packet on port %r for device %d, but "
+                "it should have arrived on one of these ports: %r.\n%s"
+                % (result.port, device_number, ports, result.format())
+            )
+            return (0, None)
+
+    if isinstance(result, test.dataplane.PollFailure):
+        return (-1, None)
+
+    return (0, None)
 
 class FgEcmpTest(BaseTest):
 
@@ -44,7 +102,7 @@ def log(self, message):
 
 
     def trigger_mac_learning(self, serv_ports):
-      	for src_port in serv_ports:
+        for src_port in serv_ports:
             pkt = simple_eth_packet(
                 eth_dst=self.router_mac,
                 eth_src=self.dataplane.get_mac(0, src_port),
@@ -154,11 +212,15 @@ def fg_ecmp(self):
         tuple_to_port_map ={}
         hit_count_map = {}
 
-        if not os.path.exists(PERSIST_MAP):
+        if not os.path.exists(PERSIST_MAP) and self.test_case == 'create_flows':
             with open(PERSIST_MAP, 'w'): pass
-        else:
+        elif not self.test_case == 'verify_packets_received':
             with open(PERSIST_MAP) as fp:
-                tuple_to_port_map = json.load(fp)
+                try:
+                    tuple_to_port_map = json.load(fp)
+                except ValueError:
+                    print 'Decoding JSON failed for persist map'
+                    assert False
 
         if tuple_to_port_map is None or self.dst_ip not in tuple_to_port_map:
             tuple_to_port_map[self.dst_ip] = {}
@@ -188,6 +250,7 @@ def fg_ecmp(self):
                 (port_idx, _) = self.send_rcv_ip_pkt(
                     in_port, src_port, dst_port, src_ip, dst_ip, self.serv_ports, ipv4)
                 assert port_idx == port
+                tuple_to_port_map[self.dst_ip][src_ip] = port_idx
             return
 
         elif self.test_case == 'hash_check_warm_boot':
@@ -199,14 +262,33 @@ def fg_ecmp(self):
                 else:
                     in_port = self.net_ports[0]
                 (port_idx, _) = self.send_rcv_ip_pkt_warm(
-                    in_port, src_port, dst_port, src_ip, dst_ip, self.serv_ports, ipv4, port)
+                    in_port, src_port, dst_port, src_ip, dst_ip, port, ipv4)
                 if port_idx == -1:
                     total_flood_pkts = total_flood_pkts + 1
             # Ensure that flooding duration in warm reboot is less than 10% of total packet count
             self.log("Number of flood packets were: " + str(total_flood_pkts))
             assert (total_flood_pkts < (0.1 * len(tuple_to_port_map[self.dst_ip])))
             return
 
+        elif self.test_case == 'verify_packets_received':
+            self.log("Ensure that all packets were received ...")
+            total_num_pkts_lost = 0
+            for i in range(0, self.num_flows):
+                src_ip = str(base_ip + i)
+                if self.inner_hashing:
+                    in_port = random.choice(self.net_ports)
+                else:
+                    in_port = self.net_ports[0]
+                (port_idx, _) = self.send_rcv_ip_pkt_lossy(
+                    in_port, src_port, dst_port, src_ip, dst_ip, self.serv_ports, ipv4)
+                if port_idx == -1:
+                    total_num_pkts_lost = total_num_pkts_lost + 1
+
+            self.log("Number of lost packets were: " + str(total_num_pkts_lost))
+            # Ensure less than 1% packet loss
+            assert (total_num_pkts_lost < (MAX_ONE_PERCENT_LOSS * self.num_flows))
+            return
+
         elif self.test_case == 'bank_check':
             self.log("Send the same flows once again and verify that they end up on the same bank...")
             for src_ip, port in tuple_to_port_map[self.dst_ip].iteritems():
@@ -333,47 +415,32 @@ def fg_ecmp(self):
         return
 
 
-    def verify_packet_warm(test, pkt, port, device_number=0, timeout=None, n_timeout=None):
-        # This packet verification function accounts for possible flood during warm boot
-        # We ensure that packets are received on the expected port, and return a special
-        # return value of -1 to denote that a flood had occured. The caller can use the 
-        # special return value to identify how many packets were flooded. 
-
-        if timeout is None:
-            timeout = ptf.ptfutils.default_timeout
-        if n_timeout is None:
-            n_timeout = ptf.ptfutils.default_negative_timeout
-        logging.debug("Checking for pkt on device %d, port %r", device_number, port)
-        result = dp_poll(test, device_number=device_number, timeout=timeout, exp_pkt=pkt)
-        verify_no_other_packets(test, device_number=device_number, timeout=n_timeout)
-
-        if isinstance(result, test.dataplane.PollSuccess):
-            if result.port != port:
-                # Flood case, check if packet rcvd on expected port as well
-                verify_packet(test, pkt, port)
-                return (-1, None)
-            else:
-                return (port, result.packet)
+    def send_rcv_ip_pkt_lossy(self, in_port, sport, dport, src_ip_addr, dst_ip_addr,
+                            exp_port, ipv4=True):
 
-        assert(isinstance(result, test.dataplane.PollFailure))
-        test.fail("Did not receive expected packet on any of ports %r for device %d.\n%s"
-                    % (ports, device_number, result.format()))
-        return (0, None)
+        if ipv4:
+            (matched_index, received) = self.send_rcv_ipv4_pkt(in_port, sport, dport,
+                    src_ip_addr, dst_ip_addr, exp_port, verify_packet_any_port_lossy)
+        else:
+            (matched_index, received) = self.send_rcv_ipv6_pkt(in_port, sport, dport,
+                    src_ip_addr, dst_ip_addr, exp_port, verify_packet_any_port_lossy)
+
+        return (matched_index, received)
 
 
     def send_rcv_ip_pkt_warm(self, in_port, sport, dport, src_ip_addr, dst_ip_addr,
-                       dst_port_list, ipv4=True, exp_port=None):
+                            exp_port, ipv4=True):
 
         # Simulate bidirectional traffic for mac learning, since mac learning(fdb) is flushed
         # as part of warm reboot
         self.trigger_mac_learning([exp_port])
 
         if ipv4:
             (matched_index, received) = self.send_rcv_ipv4_pkt(in_port, sport, dport,
-                    src_ip_addr, dst_ip_addr, dst_port_list, exp_port)
+                    src_ip_addr, dst_ip_addr, exp_port, verify_packet_warm)
         else:
             (matched_index, received) = self.send_rcv_ipv6_pkt(in_port, sport, dport,
-                    src_ip_addr, dst_ip_addr, dst_port_list, exp_port)
+                    src_ip_addr, dst_ip_addr, exp_port, verify_packet_warm)
 
         return (matched_index, received)
 
@@ -383,10 +450,10 @@ def send_rcv_ip_pkt(self, in_port, sport, dport, src_ip_addr, dst_ip_addr,
 
         if ipv4:
             (matched_index, received) = self.send_rcv_ipv4_pkt(in_port, sport, dport,
-                    src_ip_addr, dst_ip_addr, dst_port_list)
+                    src_ip_addr, dst_ip_addr, dst_port_list, verify_packet_any_port)
         else:
             (matched_index, received) = self.send_rcv_ipv6_pkt(in_port, sport, dport,
-                    src_ip_addr, dst_ip_addr, dst_port_list)
+                    src_ip_addr, dst_ip_addr, dst_port_list, verify_packet_any_port)
 
         assert received
 
@@ -397,7 +464,7 @@ def send_rcv_ip_pkt(self, in_port, sport, dport, src_ip_addr, dst_ip_addr,
 
 
     def send_rcv_ipv4_pkt(self, in_port, sport, dport,
-                         ip_src, ip_dst, dst_port_list, exp_port=None):
+                         ip_src, ip_dst, dst_port_list, verify_fn):
         src_mac = self.dataplane.get_mac(0, in_port)
         rand_int = random.randint(1, 254)
 
@@ -431,14 +498,11 @@ def send_rcv_ipv4_pkt(self, in_port, sport, dport,
         masked_exp_pkt.set_do_not_care_scapy(scapy.IP, "chksum")
         masked_exp_pkt.set_do_not_care_scapy(scapy.IP, "ttl")
 
-        if exp_port is None:
-            return verify_packet_any_port(self, masked_exp_pkt, dst_port_list)
-        else:
-            return self.verify_packet_warm(masked_exp_pkt, exp_port)
+        return verify_fn(self, masked_exp_pkt, dst_port_list)
 
 
     def send_rcv_ipv6_pkt(self, in_port, sport, dport,
-                         ip_src, ip_dst, dst_port_list, exp_port=None):
+                         ip_src, ip_dst, dst_port_list, verify_fn):
         src_mac = self.dataplane.get_mac(0, in_port)
         rand_int = random.randint(1, 254)
 
@@ -478,11 +542,7 @@ def send_rcv_ipv6_pkt(self, in_port, sport, dport,
         masked_exp_pkt.set_do_not_care_scapy(scapy.Ether, "src")
         masked_exp_pkt.set_do_not_care_scapy(scapy.IPv6, "hlim")
 
-        if exp_port is None:
-            return verify_packet_any_port(self, masked_exp_pkt, dst_port_list)
-        else:
-            return self.verify_packet_warm(masked_exp_pkt, exp_port)
-
+        return verify_fn(self, masked_exp_pkt,dst_port_list)
 
     def runTest(self):
         # Main function which triggers all the tests

tests/ecmp/test_fgnhg.py

@@ -12,6 +12,7 @@
 from tests.common.fixtures.ptfhost_utils import copy_ptftests_directory   # lgtm[py/unused-import]
 from tests.common.fixtures.ptfhost_utils import change_mac_addresses      # lgtm[py/unused-import]
 from tests.common.fixtures.ptfhost_utils import remove_ip_addresses       # lgtm[py/unused-import]
+from tests.common.fixtures.ptfhost_utils import copy_arp_responder_py     # lgtm[py/unused-import]
 
 # Constants
 NUM_NHs = 8
@@ -126,6 +127,37 @@ def setup_neighbors(duthost, ptfhost, ip_to_port):
     duthost.shell("sonic-cfggen -j /tmp/neigh.json --write-to-db")
 
 
+def setup_arpresponder(ptfhost, ip_to_port):
+    logger.info("Copy arp_responder to ptfhost")
+    # Stop existing arp responder if running
+    ptfhost.command('supervisorctl stop arp_responder')
+
+    d = defaultdict(list)
+
+    for ip, port in ip_to_port.items():
+        iface = "eth{}".format(port)
+        d[iface].append(ip)
+
+    with open('/tmp/from_t1.json', 'w') as file:
+        json.dump(d, file)
+
+    ptfhost.copy(src='/tmp/from_t1.json', dest='/tmp/from_t1.json')
+
+    extra_vars = {
+            'arp_responder_args': ''
+    }
+
+    ptfhost.host.options['variable_manager'].extra_vars.update(extra_vars)
+    ptfhost.template(src='templates/arp_responder.conf.j2', dest='/tmp')
+    ptfhost.command("cp /tmp/arp_responder.conf.j2 /etc/supervisor/conf.d/arp_responder.conf")
+
+    ptfhost.command('supervisorctl reread')
+    ptfhost.command('supervisorctl update')
+
+    logger.info("Start arp_responder")
+    ptfhost.command('supervisorctl start arp_responder')
+
+
 def create_fg_ptf_config(ptfhost, ip_to_port, port_list, bank_0_port, bank_1_port, router_mac, net_ports):
     fg_ecmp = {
             "serv_ports": port_list,
@@ -144,8 +176,8 @@ def create_fg_ptf_config(ptfhost, ip_to_port, port_list, bank_0_port, bank_1_por
 def setup_test_config(duthost, ptfhost, cfg_facts, router_mac, net_ports, vlan_ip):
     port_list, ip_to_port, bank_0_port, bank_1_port = configure_interfaces(cfg_facts, duthost, ptfhost, vlan_ip)
     generate_fgnhg_config(duthost, ip_to_port, bank_0_port, bank_1_port)
+    setup_arpresponder(ptfhost, ip_to_port)
     time.sleep(60)
-    setup_neighbors(duthost, ptfhost, ip_to_port)
     create_fg_ptf_config(ptfhost, ip_to_port, port_list, bank_0_port, bank_1_port, router_mac, net_ports)
     return port_list, ip_to_port, bank_0_port, bank_1_port
 
@@ -174,6 +206,41 @@ def partial_ptf_runner(ptfhost, test_case, dst_ip, exp_flow_count, **kwargs):
             log_file=log_file)
 
 
+def validate_packet_flow_without_neighbor_resolution(ptfhost, duthost, ip_to_port, prefix_list):
+    logger.info("Validating packet flow of fine grained ecmp without neighbor resolution")
+    # Init base test params
+    if isinstance(ipaddress.ip_network(prefix_list[0]), ipaddress.IPv4Network):
+        ipcmd = "ip route"
+    else:
+        ipcmd = "ipv6 route"
+
+    vtysh_base_cmd = "vtysh -c 'configure terminal'"
+    vtysh_base_cmd = duthost.get_vtysh_cmd_for_namespace(vtysh_base_cmd, DEFAULT_NAMESPACE)
+    dst_ip = prefix_list[0].split('/')[0]
+
+    cmd = vtysh_base_cmd
+    for nexthop in ip_to_port:
+        cmd = cmd + " -c '{} {} {}'".format(ipcmd, prefix_list[0], nexthop)
+    configure_dut(duthost, cmd)
+
+    # Validate packet flow works
+    partial_ptf_runner(ptfhost, 'verify_packets_received', dst_ip, [])
+
+    # Validate that neigh was resolved as part of packet flow
+    if isinstance(ipaddress.ip_network(prefix_list[0]), ipaddress.IPv4Network):
+        show_neigh = duthost.shell("show arp")['stdout']
+    else:
+        show_neigh = duthost.shell("show ndp")['stdout']
+
+    neigh_resolved = False
+
+    for nexthop in ip_to_port:
+        if nexthop in show_neigh:
+            neigh_resolved = True
+            break
+    assert neigh_resolved
+
+
 def fg_ecmp(ptfhost, duthost, router_mac, net_ports, port_list, ip_to_port, bank_0_port, bank_1_port, prefix_list):
 
     # Init base test params
@@ -519,10 +586,14 @@ def test_fg_ecmp(common_setup_teardown, ptfhost):
 
     # IPv4 test
     port_list, ipv4_to_port, bank_0_port, bank_1_port = setup_test_config(duthost, ptfhost, cfg_facts, router_mac, net_ports, DEFAULT_VLAN_IPv4)
+    validate_packet_flow_without_neighbor_resolution(ptfhost, duthost, ipv4_to_port, PREFIX_IPV4_LIST)
+    setup_neighbors(duthost, ptfhost, ipv4_to_port)
     fg_ecmp(ptfhost, duthost, router_mac, net_ports, port_list, ipv4_to_port, bank_0_port, bank_1_port, PREFIX_IPV4_LIST)
     fg_ecmp_to_regular_ecmp_transitions(ptfhost, duthost, router_mac, net_ports, port_list, ipv4_to_port, bank_0_port, bank_1_port, PREFIX_IPV4_LIST, cfg_facts)
 
     # IPv6 test
     port_list, ipv6_to_port, bank_0_port, bank_1_port = setup_test_config(duthost, ptfhost, cfg_facts, router_mac, net_ports, DEFAULT_VLAN_IPv6)
+    validate_packet_flow_without_neighbor_resolution(ptfhost, duthost, ipv6_to_port, PREFIX_IPV6_LIST)
+    setup_neighbors(duthost, ptfhost, ipv6_to_port)
     fg_ecmp(ptfhost, duthost, router_mac, net_ports, port_list, ipv6_to_port, bank_0_port, bank_1_port, PREFIX_IPV6_LIST)
     fg_ecmp_to_regular_ecmp_transitions(ptfhost, duthost, router_mac, net_ports, port_list, ipv6_to_port, bank_0_port, bank_1_port, PREFIX_IPV6_LIST, cfg_facts)