[advanced-reboot] Test VLAN gateway address resolvability during upgrade

ansible/roles/test/files/ptftests/py3/advanced-reboot.py

@@ -228,6 +228,7 @@ def __init__(self):
         # Default settings
         self.ping_dut_pkts = 10
         self.arp_ping_pkts = 1
+        self.arp_vlan_gw_ping_pkts = 10
         self.nr_pc_pkts = 100
         self.nr_tests = 3
         self.reboot_delay = 10
@@ -251,6 +252,7 @@ def __init__(self):
         self.cpu_state = StateMachine('init')
         self.asic_state = StateMachine('init')
         self.vlan_state = StateMachine('init')
+        self.vlan_gw_state = StateMachine('init')
         self.vlan_lock = threading.RLock()
         self.asic_state_time = {}  # Recording last asic state entering time
         self.asic_vlan_reach = []  # Recording asic vlan reachability
@@ -708,6 +710,7 @@ def setUp(self):
         self.generate_from_vlan()
         self.generate_ping_dut_lo()
         self.generate_arp_ping_packet()
+        self.generate_arp_vlan_gw_packets()
 
         if 'warm-reboot' in self.reboot_type:
             self.log(self.get_sad_info())
@@ -876,6 +879,20 @@ def generate_ping_dut_lo(self):
         self.ping_dut_exp_packet.set_do_not_care_scapy(scapy.IP, "id")
         self.ping_dut_exp_packet.set_do_not_care_scapy(scapy.IP, "chksum")
 
+    def calc_offset_and_size(self, packet, layer, field):
+        """
+        Calculate the offset and size of a field, in a packet. Return the offset and size
+        as a tuple, both in bits. Return -1, 0 if the field cannot be found.
+        """
+        offset = 0
+        while packet:  # for each payload
+            for fld in packet.fields_desc:  # for each field
+                if fld.name == field and isinstance(packet, layer):
+                    return int(offset) * 8, fld.i2len(packet, packet.getfieldval(fld.name)) * 8
+                offset += fld.i2len(packet, packet.getfieldval(fld.name))  # add length
+            packet = packet.payload
+        return -1, 0
+
     def generate_arp_ping_packet(self):
         vlan = next(k for k, v in self.ports_per_vlan.items() if v)
         vlan_ip_range = self.vlan_ip_range[vlan]
@@ -902,10 +919,45 @@ def generate_arp_ping_packet(self):
         self.arp_ping = bytes(packet)
         self.arp_resp = Mask(expect)
         self.arp_resp.set_do_not_care_scapy(scapy.Ether, 'src')
-        self.arp_resp.set_do_not_care_scapy(scapy.ARP,   'hwtype')
-        self.arp_resp.set_do_not_care_scapy(scapy.ARP,   'hwsrc')
+        self.arp_resp.set_do_not_care(*self.calc_offset_and_size(expect, scapy.ARP, "hwsrc"))
         self.arp_src_port = src_port
 
+    def generate_arp_vlan_gw_packets(self):
+        self.arp_vlan_gw_ping_packets = []
+
+        for src_port in self.active_port_indices if self.is_dualtor else self.vlan_host_ping_map:
+            src_addr = random.choice(list(self.vlan_host_ping_map[src_port].keys()))
+            src_mac = self.hex_to_mac(
+                self.vlan_host_ping_map[src_port][src_addr])
+            packet = simple_arp_packet(eth_src=src_mac,
+                                       arp_op=1,
+                                       ip_snd=src_addr,
+                                       ip_tgt="192.168.0.1",  # TODO: make this dynamic
+                                       hw_snd=src_mac)
+
+            self.arp_vlan_gw_ping_packets.append((src_port, bytes(packet)))
+
+        exp_packet = simple_arp_packet(pktlen=42, eth_src=self.vlan_mac,
+                                       arp_op=2,
+                                       ip_snd="192.168.0.1",
+                                       hw_snd=self.vlan_mac)
+        self.arp_vlan_gw_ping_exp_packet = Mask(exp_packet, ignore_extra_bytes=True)
+        self.arp_vlan_gw_ping_exp_packet.set_do_not_care_scapy(scapy.Ether, 'dst')
+        # PTF's field size calculation is broken for dynamic length fields, do it ourselves
+        self.arp_vlan_gw_ping_exp_packet.set_do_not_care(*self.calc_offset_and_size(exp_packet, scapy.ARP, "pdst"))
+        self.arp_vlan_gw_ping_exp_packet.set_do_not_care(*self.calc_offset_and_size(exp_packet, scapy.ARP, "hwdst"))
+
+        exp_packet = simple_arp_packet(pktlen=42, eth_src=self.vlan_mac,
+                                       arp_op=2,
+                                       ip_snd="192.168.0.1",
+                                       hw_snd=self.vlan_mac)
+        exp_packet = exp_packet / ("fe11e1" * 6)
+        self.arp_vlan_gw_ferret_exp_packet = Mask(exp_packet)
+        self.arp_vlan_gw_ferret_exp_packet.set_do_not_care_scapy(scapy.Ether, 'dst')
+        # PTF's field size calculation is broken for dynamic length fields, do it ourselves
+        self.arp_vlan_gw_ferret_exp_packet.set_do_not_care(*self.calc_offset_and_size(exp_packet, scapy.ARP, "pdst"))
+        self.arp_vlan_gw_ferret_exp_packet.set_do_not_care(*self.calc_offset_and_size(exp_packet, scapy.ARP, "hwdst"))
+
     def put_nowait(self, queue, data):
         try:
             queue.put_nowait(data)
@@ -1689,7 +1741,8 @@ def send_in_background(self, packets_list=None):
             #  2. during warm neighbor restoration DUT will send a lot of ARP requests which we are not interested in
             # This is essential to get stable results
             self.apply_filter_all_ports(
-                'not (arp and ether src {}) and not tcp'.format(self.test_params['dut_mac']))
+                'not (arp and ether src {} and ether dst ff:ff:ff:ff:ff:ff) and not tcp'.format(
+                    self.test_params['dut_mac']))
             sender_start = datetime.datetime.now()
             self.log("Sender started at %s" % str(sender_start))
 
@@ -2330,6 +2383,21 @@ def log_vlan_state_change(self, reachable):
             self.log("VLAN ARP state transition from %s to %s" % (old, state))
             self.vlan_state.set(state)
 
+    def log_vlan_gw_state_change(self, reachable, partial=False, flooding=False):
+        old = self.vlan_gw_state.get()
+
+        if reachable:
+            state = 'up' if not partial else 'partial'
+        else:
+            state = 'down'
+
+        self.vlan_gw_state.set_flooding(flooding)
+
+        if old != state:
+            self.log("VLAN GW state transition from %s to %s" %
+                     (old, state))
+            self.vlan_gw_state.set(state)
+
     def reachability_watcher(self):
         # This function watches the reachability of the CPU port, and ASIC. It logs the state
         # changes for future analysis
@@ -2352,6 +2420,7 @@ def reachability_watcher(self):
                 self.dataplane_io_lock.release()
             else:
                 self.log("Reachability watcher - Dataplane is busy. Skipping the check")
+
             self.log('Reachability watcher - checking control plane')
             total_rcv_pkt_cnt = self.pingDut()
             reachable = total_rcv_pkt_cnt > 0 and total_rcv_pkt_cnt > self.ping_dut_pkts * 0.7
@@ -2361,6 +2430,14 @@ def reachability_watcher(self):
             total_rcv_pkt_cnt = self.arpPing()
             reachable = total_rcv_pkt_cnt >= self.arp_ping_pkts
             self.log_vlan_state_change(reachable)
+
+            self.log('Reachability watcher - checking VLAN GW IP')
+            total_rcv_pkt_cnt = self.arpVlanGwPing()
+            reachable = total_rcv_pkt_cnt > 0 and total_rcv_pkt_cnt > self.arp_vlan_gw_ping_pkts * 0.7
+            partial = total_rcv_pkt_cnt > 0 and total_rcv_pkt_cnt < self.arp_vlan_gw_ping_pkts
+            flooding = reachable and total_rcv_pkt_cnt > self.arp_vlan_gw_ping_pkts
+            self.log_vlan_gw_state_change(reachable, partial, flooding)
+
             self.watcher_is_running.set()   # Watcher is running.
         self.log('Reachability watcher stopped')
         self.watcher_is_stopped.set()       # Watcher has stopped.
@@ -2416,3 +2493,15 @@ def arpPing(self):
         self.log("Send %5d Received %5d arp ping" %
                  (self.arp_ping_pkts, total_rcv_pkt_cnt), True)
         return total_rcv_pkt_cnt
+
+    def arpVlanGwPing(self):
+        total_rcv_pkt_cnt = 0
+        packets = random.sample(self.arp_vlan_gw_ping_packets, self.arp_vlan_gw_ping_pkts)
+        for packet in packets:
+            src_port, arp_packet = packet
+            testutils.send_packet(self, src_port, arp_packet)
+        total_rcv_pkt_cnt = testutils.count_matched_packets_all_ports(
+            self, self.arp_vlan_gw_ping_exp_packet, self.vlan_ports, timeout=self.PKT_TOUT)
+        self.log("Send %5d Received %5d arp vlan gw ping" %
+                 (self.arp_vlan_gw_ping_pkts, total_rcv_pkt_cnt), True)
+        return total_rcv_pkt_cnt

tests/arp/files/ferret.py

@@ -174,7 +174,7 @@ class Responder(object):
     def __init__(self, db, vxlan_port):
         # defines a part of the packet for ARP Reply
         self.arp_chunk = binascii.unhexlify('08060001080006040002')
-        self.arp_pad = binascii.unhexlify('00' * 18)
+        self.arp_pad = binascii.unhexlify('fe11e1' * 6)
         self.db = db
         self.vxlan_port = vxlan_port
 

tests/common/fixtures/advanced_reboot.py

@@ -538,12 +538,13 @@ def print_test_logs_summary(self, log_dir):
             if log_file.endswith('reboot.log'):
                 with open(os.path.join(log_dir, log_file)) as reboot_log:
                     reboot_text_log_file = reboot_log.read()
-                    reboot_summary = re.search(r"Summary:(\n|.)*?=========", reboot_text_log_file).group()
-                    if reboot_summary.find('Fails') == -1:
-                        # if no fails detected - the test passed, print the summary only
-                        logger.info('\n'+reboot_summary)
-                    else:
-                        logger.info(reboot_text_log_file)
+                    reboot_summary = re.search(r"Summary:(\n|.)*?=========", reboot_text_log_file)
+                    if reboot_summary:
+                        if reboot_summary.group().find('Fails') == -1:
+                            # if no fails detected - the test passed, print the summary only
+                            logger.info('\n'+reboot_summary.group())
+                        else:
+                            logger.info(reboot_text_log_file)
 
     def acl_manager_checker(self, error_list):
         """
@@ -734,6 +735,7 @@ def __runPtfRunner(self, rebootOper=None):
             "service_list": None if self.rebootType != 'service-warm-restart' else self.service_list,
             "service_data": None if self.rebootType != 'service-warm-restart' else self.service_data,
             "neighbor_type": self.neighborType,
+            "kvm_support": True,
         }
 
         if self.dual_tor_mode: