upgrade ceos to version 4.25.5.1M, fix the issue of arista endless loop

ansible/group_vars/all/ceos.yml

@@ -1,4 +1,8 @@
-ceos_image_filename: cEOS64-lab-4.23.2F.tar.xz
-ceos_image_orig: ceosimage:4.23.2F
-ceos_image: ceosimage:4.23.2F-1
+# upgrade cEOS to 4.25.5.1M from 4.23.2F at 2021/09/15
+#ceos_image_filename: cEOS64-lab-4.23.2F.tar.xz
+#ceos_image_orig: ceosimage:4.23.2F
+#ceos_image: ceosimage:4.23.2F-1
+ceos_image_filename: cEOS64-lab-4.25.5.1M.tar
+ceos_image_orig: ceosimage:4.25.5.1M
+ceos_image: ceosimage:4.25.5.1M-1
 skip_ceos_image_downloading: false

ansible/roles/eos/tasks/ceos_ensure_reachable.yml

@@ -1,7 +1,7 @@
 - block:
   - name: set time out threshold
     set_fact:
-      timeout_threshold: 120
+      timeout_threshold: 240
 
   - name: wait until container's mgmt-ip is reachable
     wait_for:

ansible/roles/test/files/ptftests/arista.py

@@ -52,6 +52,8 @@ def __init__(self, ip, queue, test_params, log_cb=None, login='admin', password=
         self.min_bgp_gr_timeout = int(test_params['min_bgp_gr_timeout'])
         self.reboot_type = test_params['reboot_type']
         self.bgp_v4_v6_time_diff = test_params['bgp_v4_v6_time_diff']
+        # unit: second
+        self.ssh_cmd_timeout = 10
 
     def __del__(self):
         self.disconnect()
@@ -65,6 +67,10 @@ def connect(self):
         self.conn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
         self.conn.connect(self.ip, username=self.login, password=self.password, allow_agent=False, look_for_keys=False)
         self.shell = self.conn.invoke_shell()
+        # avoid paramiko Channel.recv() stuck forever
+        self.shell.settimeout(self.ssh_cmd_timeout)
+        # avoid garbage collection from socket
+        self.shell.keep_this = self.conn
 
         first_prompt = self.do_cmd(None, prompt = '>')
         self.arista_prompt = self.get_arista_prompt(first_prompt)
@@ -83,17 +89,75 @@ def get_arista_prompt(self, first_prompt):
         # match all modes - A#, A(config)#, A(config-if)#
         return prompt.strip().replace('>', '.*#')
 
+    def exec_command(self, cmd):
+        """
+        Execute single command without interaction
+        Args:
+            cmd (str): command, e.g. show version
+        Returns:
+            int: command execution status code
+            str: command return value
+        """
+        self.log('exec_command is: %s' % (cmd))
+        try:
+            stdin, stdout, stderr = self.conn.exec_command(cmd, timeout=self.ssh_cmd_timeout)
+            (status_code, res) = (stdout.channel.recv_exit_status(), stdout.read())
+        except Exception as err:
+            msg = 'Receive ssh command result error: cmd={} msg={} type={}'.format(cmd, err, type(err))
+            self.log(msg)
+            return 1, msg
+        self.log('exec_command return is, status: %d, value: |%s|' % (status_code, res))
+        return status_code, res
+
+    def exec_command_compatible(self, *cmds):
+        """
+        Execute several equivalent commands:
+            return first success command returned value,
+            return last command returned value if all failed.
+        Used for compatible scenario, for example:
+            in EOS 4.20.15M lacp show command is 'show lacp neighbor', but in EOS 4.25.5.1M it is 'show lacp peer'
+        Args:
+            cmds (*str): command, e.g. 'show lacp neighbor', 'show lacp peer'
+        Returns:
+            str: command return value
+        """
+        self.log('exec_command_compatible is: %s' % (str(cmds)))
+        value = None
+        for cmd in cmds:
+            (code, value) = self.exec_command(cmd)
+            if code == 0:
+                break
+        self.log('exec_command_compatible return is: %s' % (value))
+        return value
+
+
     def do_cmd(self, cmd, prompt = None):
+        self.log('do_cmd is: %s' % (cmd))
         if prompt == None:
             prompt = self.arista_prompt
 
         if cmd is not None:
             self.shell.send(cmd + '\n')
-
+        # wait 0.2s for the executing
+        time.sleep(0.2)
         input_buffer = ''
         while re.search(prompt, input_buffer) is None:
-            input_buffer += self.shell.recv(16384)
+            if not self.shell.recv_ready():
+                time.sleep(0.5)
+                continue
+            try:
+                input_buffer += self.shell.recv(16384)
+            except Exception as err:
+                msg = 'Receive ssh command result error: cmd={} msg={} type={}'.format(cmd, err, type(err))
+                self.log(msg)
+                return msg
 
+            # cEOS will not return an arista_prompt if you send lots of 'exit' to close the ssh connect(vEOS do will),
+            # then an endless loop emerges,
+            # so if input_buffer is merely an 'exit', we can break the loop immediately
+            if input_buffer.replace('\n', '').replace('\r', '').strip().lower() == 'exit':
+                break
+        self.log('do_cmd return is: %s' % (input_buffer))
         return input_buffer
 
     def disconnect(self):
@@ -124,33 +188,41 @@ def run(self):
 
         while not (quit_enabled and v4_routing_ok and v6_routing_ok):
             cmd = None
+            self.log('quit_enabled is: %s' % (quit_enabled))
             # quit command was received, we don't process next commands
             # but wait for v4_routing_ok and v6_routing_ok
             if not quit_enabled:
                 cmd = self.queue.get()
+                self.log('self.queue.get() finish: %s'%(cmd))
                 if cmd == 'quit':
                     quit_enabled = True
                     continue
 
             cur_time = time.time()
             info = {}
             debug_info = {}
-            lacp_output = self.do_cmd('show lacp neighbor')
+            # 'show lacp neighbor' command is deprecated by 'show lacp peer'
+            # execute both of them, get the success returned one as lacp_output
+            lacp_output = self.exec_command_compatible('show lacp neighbor', 'show lacp peer')
             info['lacp'] = self.parse_lacp(lacp_output)
+            self.log('lacp is: %s' % (info['lacp']))
             bgp_neig_output = self.do_cmd('show ip bgp neighbors')
             info['bgp_neig'] = self.parse_bgp_neighbor(bgp_neig_output)
+            self.log('bgp_neig is: %s' % (str(info['bgp_neig'])))
 
             v4_routing, bgp_route_v4_output = self.check_bgp_route(self.v4_routes)
             if v4_routing != v4_routing_ok:
                 v4_routing_ok = v4_routing
                 self.log('BGP routing for ipv4 OK: %s' % (v4_routing_ok))
             info['bgp_route_v4'] = v4_routing_ok
+            self.log('bgp_route_v4 is: %s' % (str(info['bgp_route_v4'])))
 
             v6_routing, bgp_route_v6_output = self.check_bgp_route(self.v6_routes, ipv6=True)
             if v6_routing != v6_routing_ok:
                 v6_routing_ok = v6_routing
                 self.log('BGP routing for ipv6 OK: %s' % (v6_routing_ok))
             info["bgp_route_v6"] = v6_routing_ok
+            self.log('bgp_route_v6 is: %s' % (info['bgp_route_v6']))
 
             portchannel_output = self.do_cmd("show interfaces po1 | json")
             portchannel_output = "\n".join(portchannel_output.split("\r\n")[1:-1])
@@ -170,7 +242,7 @@ def run(self):
             samples[cur_time] = sample
             if self.DEBUG:
                 debug_data[cur_time] = {
-                    'show lacp neighbor' : lacp_output,
+                    'show lacp neighbor/peer' : lacp_output,
                     'show ip bgp neighbors' : bgp_neig_output,
                     'show ip route bgp' : bgp_route_v4_output,
                     'show ipv6 route bgp' : bgp_route_v6_output,
@@ -367,9 +439,11 @@ def parse_bgp_neighbor(self, output):
         gr_timer = None
         for line in output.split('\n'):
             if 'Restart timer is' in line:
+                self.log('bgp_neig line is: |%s|' % (line))
                 gr_active = 'is active' in line
                 gr_timer = str(line[-9:-1])
-
+                self.log('bgp_neig1 gr_active: %s, gr_timer: |%s|' % (gr_active, gr_timer))
+        self.log('bgp_neig2 gr_active: %s, gr_timer: |%s|' % (gr_active, gr_timer))
         return gr_active, gr_timer
 
     def parse_bgp_route(self, output, expects):