[warm-reboot] Routing change (add/del) during warm boot automation

ansible/roles/test/files/ptftests/advanced-reboot.py

@@ -121,7 +121,7 @@ def __init__(self):
         self.logs_info = {}
         self.log_lock = threading.RLock()
         self.vm_handle = None
-        self.pre_handle = None
+        self.sad_handle = None
         self.test_params = testutils.test_params_get()
         self.check_param('verbose', False, required=False)
         self.check_param('dut_username', '', required=True)
@@ -144,13 +144,23 @@ def __init__(self):
         self.check_param('dut_stabilize_secs', 30, required=False)
         self.check_param('preboot_files', None, required = False)
         self.check_param('preboot_oper', None, required = False) # preboot sad path to inject before warm-reboot
+        self.check_param('inboot_oper', None, required = False) # sad path to inject during warm-reboot
+        self.check_param('nexthop_ips', [], required = False) # nexthops for the routes that will be added during warm-reboot
         self.check_param('allow_vlan_flooding', False, required = False)
         self.check_param('sniff_time_incr', 60, required = False)
         if not self.test_params['preboot_oper'] or self.test_params['preboot_oper'] == 'None':
             self.test_params['preboot_oper'] = None
+        if not self.test_params['inboot_oper'] or self.test_params['inboot_oper'] == 'None':
+            self.test_params['inboot_oper'] = None
 
-        if self.test_params['preboot_oper'] is not None:
-           self.log_file_name = '/tmp/%s-%s.log' % (self.test_params['reboot_type'], self.test_params['preboot_oper'])
+        # initialize sad oper
+        if self.test_params['preboot_oper']:
+            self.sad_oper = self.test_params['preboot_oper']
+        else:
+            self.sad_oper = self.test_params['inboot_oper']
+
+        if self.sad_oper:
+           self.log_file_name = '/tmp/%s-%s.log' % (self.test_params['reboot_type'], self.sad_oper)
         else:
            self.log_file_name = '/tmp/%s.log' % self.test_params['reboot_type']
         self.log_fp = open(self.log_file_name, 'w')
@@ -298,7 +308,7 @@ def generate_arp_responder_conf(self, vlan_host_map):
 
     def dump_arp_responder_config(self, dump):
         # save data for arp_replay process
-        filename = "/tmp/from_t1.json" if self.preboot_oper is None else "/tmp/from_t1_%s.json" % self.preboot_oper
+        filename = "/tmp/from_t1.json" if self.sad_oper is None else "/tmp/from_t1_%s.json" % self.sad_oper
         with open(filename, "w") as fp:
             json.dump(dump, fp)
 
@@ -361,36 +371,98 @@ def populate_fail_info(self, fails):
                 self.fails[key] = set()
             self.fails[key] |= fails[key]
 
-    def get_preboot_info(self):
+    def get_sad_info(self):
         '''
-        Prepares the msg string to log when a preboot_oper is defined.
-        preboot_oper can be represented in the following ways
+        Prepares the msg string to log when a sad_oper is defined. Sad oper can be a preboot or inboot oper
+        sad_oper can be represented in the following ways
            eg. 'preboot_oper' - a single VM will be selected and preboot_oper will be applied to it
                'neigh_bgp_down:2' - 2 VMs will be selected and preboot_oper will be applied to the selected 2 VMs
                'neigh_lag_member_down:3:1' - this case is used for lag member down operation only. This indicates that
                                              3 VMs will be selected and 1 of the lag members in the porchannel will be brought down
+               'inboot_oper' - represents a routing change during warm boot (add or del of multiple routes)
+               'routing_add:10' - adding 10 routes during warm boot
         '''
         msg = ''
-        if self.preboot_oper:
-            msg = 'Preboot oper: %s ' % self.preboot_oper
-            if ':' in self.preboot_oper:
-                oper_list = self.preboot_oper.split(':')
-                msg = 'Preboot oper: %s ' % oper_list[0] # extract the preboot oper_type
+        if self.sad_oper:
+            msg = 'Sad oper: %s ' % self.sad_oper
+            if ':' in self.sad_oper:
+                oper_list = self.sad_oper.split(':')
+                msg = 'Sad oper: %s ' % oper_list[0] # extract the sad oper_type
                 if len(oper_list) > 2:
-                    # extract the number of VMs and the number of LAG members. preboot_oper will be of the form oper:no of VMS:no of lag members
+                    # extract the number of VMs and the number of LAG members. sad_oper will be of the form oper:no of VMS:no of lag members
                     msg += 'Number of sad path VMs: %s Lag member down in a portchannel: %s' % (oper_list[-2], oper_list[-1])
                 else:
-                    # extract the number of VMs. preboot_oper will be of the form oper:no of VMS
-                    msg += 'Number of sad path VMs: %s' % oper_list[-1]
+                    # inboot oper
+                    if 'routing' in self.sad_oper:
+                        msg += 'Number of ip addresses: %s' % oper_list[-1]
+                    else:
+                        # extract the number of VMs. preboot_oper will be of the form oper:no of VMS
+                        msg += 'Number of sad path VMs: %s' % oper_list[-1]
 
         return msg
 
+    def init_sad_oper(self):
+        if self.sad_oper:
+            self.log("Preboot/Inboot Operations:")
+            self.sad_handle = sp.SadTest(self.sad_oper, self.ssh_targets, self.portchannel_ports, self.vm_dut_map, self.test_params, self.dut_ssh, self.vlan_ports)
+            (self.ssh_targets, self.portchannel_ports, self.neigh_vm, self.vlan_ports), (log_info, fails) = self.sad_handle.setup()
+            self.populate_fail_info(fails)
+            for log in log_info:
+                self.log(log)
+
+            if self.sad_oper:
+                log_info, fails = self.sad_handle.verify()
+                self.populate_fail_info(fails)
+                for log in log_info:
+                    self.log(log)
+                self.log(" ")
+
+    def do_inboot_oper(self):
+        '''
+        Add or del routes during boot
+        '''
+        if self.sad_oper and 'routing' in self.sad_oper:
+            self.log("Performing inboot operation")
+            log_info, fails = self.sad_handle.route_setup()
+            self.populate_fail_info(fails)
+            for log in log_info:
+                self.log(log)
+            self.log(" ")
+
+    def check_inboot_sad_status(self):
+        if 'routing_add' in self.sad_oper:
+            self.log('Verify if new routes added during warm reboot are received')
+        else:
+            self.log('Verify that routes deleted during warm reboot are removed')
+
+        log_info, fails = self.sad_handle.verify(pre_check=False, inboot=True)
+        self.populate_fail_info(fails)
+        for log in log_info:
+            self.log(log)
+        self.log(" ")
+
+    def check_postboot_sad_status(self):
+        self.log("Postboot checks:")
+        log_info, fails = self.sad_handle.verify(pre_check=False, inboot=False)
+        self.populate_fail_info(fails)
+        for log in log_info:
+            self.log(log)
+        self.log(" ")
+
+    def sad_revert(self):
+        self.log("Revert to preboot state:")
+        log_info, fails = self.sad_handle.revert()
+        self.populate_fail_info(fails)
+        for log in log_info:
+            self.log(log)
+        self.log(" ")
+
     def setUp(self):
         self.fails['dut'] = set()
         self.port_indices = self.read_port_indices()
         self.portchannel_ports = self.read_portchannel_ports()
         self.vlan_ports = self.read_vlan_ports()
-        if self.test_params['preboot_oper'] is not None:
+        if self.sad_oper:
             self.build_peer_mapping()
             self.test_params['vlan_if_port'] = self.build_vlan_if_port_mapping()
 
@@ -399,7 +471,6 @@ def setUp(self):
 
         self.limit = datetime.timedelta(seconds=self.test_params['reboot_limit_in_seconds'])
         self.reboot_type = self.test_params['reboot_type']
-        self.preboot_oper = self.test_params['preboot_oper']
         if self.reboot_type not in ['fast-reboot', 'warm-reboot']:
             raise ValueError('Not supported reboot_type %s' % self.reboot_type)
         self.dut_ssh = self.test_params['dut_username'] + '@' + self.test_params['dut_hostname']
@@ -415,18 +486,7 @@ def setUp(self):
                 self.ssh_targets.append(vm)
 
         self.log("Converted addresses VMs: %s" % str(self.ssh_targets))
-        if self.preboot_oper is not None:
-            self.log("Preboot Operations:")
-            self.pre_handle = sp.PrebootTest(self.preboot_oper, self.ssh_targets, self.portchannel_ports, self.vm_dut_map, self.test_params, self.dut_ssh, self.vlan_ports)
-            (self.ssh_targets, self.portchannel_ports, self.neigh_vm, self.vlan_ports), (log_info, fails) = self.pre_handle.setup()
-            self.populate_fail_info(fails)
-            for log in log_info:
-                self.log(log)
-            log_info, fails = self.pre_handle.verify()
-            self.populate_fail_info(fails)
-            for log in log_info:
-                self.log(log)
-            self.log(" ")
+        self.init_sad_oper()
 
         self.vlan_host_map = self.generate_vlan_servers()
         arp_responder_conf = self.generate_arp_responder_conf(self.vlan_host_map)
@@ -458,7 +518,7 @@ def setUp(self):
         self.generate_arp_ping_packet()
 
         if self.reboot_type == 'warm-reboot':
-            self.log(self.get_preboot_info())
+            self.log(self.get_sad_info())
 
             # Pre-generate list of packets to be sent in send_in_background method.
             generate_start = datetime.datetime.now()
@@ -706,6 +766,9 @@ def runTest(self):
 
             if self.reboot_type == 'fast-reboot':
                 self.light_probe = True
+            else:
+                # add or del routes during boot
+                self.do_inboot_oper()
 
             self.reboot_start = datetime.datetime.now()
             self.log("Dut reboots: reboot start %s" % str(self.reboot_start))
@@ -803,13 +866,13 @@ def wait_for_ssh_threads():
                 self.fails['dut'].add("Dataplane didn't route to all servers, when control-plane was down: %d vs %d" % (no_cp_replies, self.nr_vl_pkts))
 
             if self.reboot_type == 'warm-reboot':
-                if self.preboot_oper is not None and self.pre_handle is not None:
-                    self.log("Postboot checks:")
-                    log_info, fails = self.pre_handle.verify(pre_check=False)
-                    self.populate_fail_info(fails)
-                    for log in log_info:
-                        self.log(log)
-                    self.log(" ")
+                # after the data plane is up, check for routing changes
+                if self.test_params['inboot_oper'] and self.sad_handle:
+                    self.check_inboot_sad_status()
+
+                # postboot check for all preboot operations
+                if self.test_params['preboot_oper'] and self.sad_handle:
+                    self.check_postboot_sad_status()
 
                 else:
                     # verify there are no interface flaps after warm boot
@@ -822,9 +885,10 @@ def wait_for_ssh_threads():
             self.watching = False
 
             # revert to pretest state
-            if self.preboot_oper is not None and self.pre_handle is not None:
-                self.log("Revert to preboot state:")
-                self.pre_handle.revert()
+            if self.sad_oper and self.sad_handle:
+                self.sad_revert()
+                if self.test_params['inboot_oper']:
+                    self.check_postboot_sad_status()
                 self.log(" ")
 
             # Generating report
@@ -1018,7 +1082,7 @@ def sniff_in_background(self, wait = None):
         self.sniffer_started.clear()
 
     def save_sniffed_packets(self):
-        filename = "/tmp/capture_%s.pcap" % self.preboot_oper if self.preboot_oper is not None else "/tmp/capture.pcap"
+        filename = "/tmp/capture_%s.pcap" % self.sad_oper if self.sad_oper is not None else "/tmp/capture.pcap"
         if self.packets:
             scapyall.wrpcap(filename, self.packets)
             self.log("Pcap file dumped to %s" % filename)
@@ -1152,7 +1216,7 @@ def examine_flow(self, filename = None):
             self.log("Gaps in forwarding not found.")
         self.log("Total incoming packets captured %d" % received_counter)
         if packets:
-            filename = '/tmp/capture_filtered.pcap' if self.preboot_oper is None else "/tmp/capture_filtered_%s.pcap" % self.preboot_oper
+            filename = '/tmp/capture_filtered.pcap' if self.sad_oper is None else "/tmp/capture_filtered_%s.pcap" % self.sad_oper
             scapyall.wrpcap(filename, packets)
             self.log("Filtered pcap dumped to %s" % filename)
 

ansible/roles/test/files/ptftests/arista.py

@@ -363,6 +363,12 @@ def get_bgp_info(self):
 
         return neigh_bgp, dut_bgp
 
+    def change_bgp_route(self, cfg_map):
+        self.do_cmd('configure')
+        for item in cfg_map:
+            self.do_cmd(item)
+        self.do_cmd('exit')
+
     def change_bgp_neigh_state(self, asn, is_up=True):
         state = ['shut', 'no shut']
         self.do_cmd('configure')
@@ -519,4 +525,3 @@ def check_change_time(self, output, entity, what):
 
         # Note: the first item is a placeholder
         return 0, change_count
-