Support check_forwarding_stop and wait_until_vlan_cpu_port_up in parallel, and no-dataplan-disruption case (#766)

* Support check_forwarding_stop and wait_until_vlan_cpu_port_up in parallel, and also support no-dataplan-disruption case
* Ping lo addr instead of vlan interface, so advanced-reboot will work on T1
* Refactor var name

Author: qiluo-msft

ansible/roles/test/files/ptftests/advanced-reboot.py

@@ -44,6 +44,7 @@
 import ptf.packet as scapy
 import thread
 import threading
+from multiprocessing.pool import ThreadPool, TimeoutError
 import os
 import signal
 import random
@@ -407,6 +408,7 @@ def __init__(self):
         self.cli_info = {}
         self.logs_info = {}
         self.log_fp = open('/tmp/reboot.log', 'w')
+        self.log_lock = threading.RLock()
         self.test_params = testutils.test_params_get()
         self.check_param('verbose', False,   required = False)
         self.check_param('dut_username', '', required = True)
@@ -492,9 +494,10 @@ def random_port(self, ports):
 
     def log(self, message, verbose=False):
         current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        if verbose and self.test_params['verbose'] or not verbose:
-            print "%s : %s" % (current_time, message)
-        self.log_fp.write("%s : %s\n" % (current_time, message))
+        with self.log_lock:
+            if verbose and self.test_params['verbose'] or not verbose:
+                print "%s : %s" % (current_time, message)
+            self.log_fp.write("%s : %s\n" % (current_time, message))
 
     def timeout(self, seconds, message):
         def timeout_exception(self, message):
@@ -528,7 +531,7 @@ def setUp(self):
         #
         self.generate_from_t1()
         self.generate_from_vlan()
-        self.generate_ping_dut_vlan_intf()
+        self.generate_ping_dut_lo()
 
         self.log("Test params:")
         self.log("DUT ssh: %s" % self.dut_ssh)
@@ -658,13 +661,14 @@ def generate_from_vlan(self):
 
         return
 
-    def generate_ping_dut_vlan_intf(self):
+    def generate_ping_dut_lo(self):
+        dut_lo_ipv4 = self.test_params['lo_prefix'].split('/')[0]
         packet = simple_icmp_packet(eth_dst=self.dut_mac,
                                     ip_src=self.from_server_src_addr,
-                                    ip_dst=self.test_params['dut_vlan_ip'])
+                                    ip_dst=dut_lo_ipv4)
 
         exp_packet = simple_icmp_packet(eth_src=self.dut_mac,
-                                        ip_src=self.test_params['dut_vlan_ip'],
+                                        ip_src=dut_lo_ipv4,
                                         ip_dst=self.from_server_src_addr,
                                         icmp_type='echo-reply')
 
@@ -708,9 +712,9 @@ def runTest(self):
             self.log("Schedule to reboot the remote switch in %s sec" % self.reboot_delay)
             thr.start()
 
-            self.log("Wait until VLAN and CPU port down")
+            self.log("Wait until CPU port down")
             self.timeout(self.task_timeout, "DUT hasn't shutdown in %d seconds" % self.task_timeout)
-            self.wait_until_vlan_cpu_port_down()
+            self.wait_until_cpu_port_down()
             self.cancel_timeout()
 
             self.reboot_start = datetime.datetime.now()
@@ -719,20 +723,32 @@ def runTest(self):
             self.log("Check that device is still forwarding Data plane traffic")
             self.assertTrue(self.check_alive(), 'DUT is not stable')
 
-            self.log("Wait until VLAN and CPU port up")
-            self.timeout(self.task_timeout, "DUT hasn't bootup in %d seconds" % self.task_timeout)
-            self.wait_until_vlan_cpu_port_up()
-            self.cancel_timeout()
+            self.log("Wait until CPU port up")
+            pool = ThreadPool(processes=10)
+            async_cpu_up = pool.apply_async(self.wait_until_cpu_port_up)
 
             self.log("Wait until ASIC stops")
-            self.timeout(self.task_timeout, "DUT hasn't stopped in %d seconds" % self.task_timeout)
-            no_routing_start, upper_replies = self.check_forwarding_stop()
-            self.cancel_timeout()
-
-            self.log("ASIC was stopped, Waiting until it's up. Stop time: %s" % str(no_routing_start))
-            self.timeout(self.task_timeout, "DUT hasn't started to work for %d seconds" % self.task_timeout)
-            no_routing_stop, _ = self.check_forwarding_resume()
-            self.cancel_timeout()
+            async_forward_stop = pool.apply_async(self.check_forwarding_stop)
+
+            try:
+                async_cpu_up.get(timeout=self.task_timeout)
+            except TimeoutError as e:
+                self.log("DUT hasn't bootup in %d seconds" % self.task_timeout)
+                raise
+
+            try:
+                no_routing_start, upper_replies = async_forward_stop.get(timeout=self.task_timeout)
+                self.log("ASIC was stopped, Waiting until it's up. Stop time: %s" % str(no_routing_start))
+            except TimeoutError:
+                self.log("ASIC never stop")
+                no_routing_start = datetime.min
+
+            if no_routing_start is not None:
+                self.timeout(self.task_timeout, "DUT hasn't started to work for %d seconds" % self.task_timeout)
+                no_routing_stop, _ = self.check_forwarding_resume()
+                self.cancel_timeout()
+            else:
+                no_routing_stop = datetime.min
 
             # wait until all bgp session are established
             self.log("Wait until bgp routing is up on all devices")
@@ -870,13 +886,13 @@ def peer_state_check(self, ip, queue):
         ssh = Arista(ip, queue, self.test_params)
         self.fails[ip], self.info[ip], self.cli_info[ip], self.logs_info[ip] = ssh.run()
 
-    def wait_until_vlan_cpu_port_down(self):
+    def wait_until_cpu_port_down(self):
         while True:
             total_rcv_pkt_cnt = self.pingDut()
             if total_rcv_pkt_cnt < self.ping_dut_pkts:
                 break
 
-    def wait_until_vlan_cpu_port_up(self):
+    def wait_until_cpu_port_up(self):
         while True:
             total_rcv_pkt_cnt = self.pingDut()
             if total_rcv_pkt_cnt >= self.ping_dut_pkts / 2: