[testbed]: support python3 and ubuntu 20.04 (#1771)

Need to move kvm testbed infrastructure to ubuntu 20.04 to support kvm save/restore, which is important to repro testbed issues.

Signed-off-by: Guohan Lu <gulv@microsoft.com>

Author: lguohan

ansible/roles/vm_set/library/ceos_network.py

@@ -207,7 +207,7 @@ def cmd(cmdline):
 
         with open(cmd_debug_fname, 'a') as fp:
             pprint("OUTPUT: %s" % stdout, fp)
-        return stdout
+        return stdout.decode('utf-8')
 
     @staticmethod
     def get_ovs_br_ports(bridge):

ansible/roles/vm_set/library/kvm_port.py

@@ -29,17 +29,17 @@ def main():
 
     try:
         output = subprocess.check_output(
-                "virsh domiflist %s" % vmname, 
-                env={"LIBVIRT_DEFAULT_URI": "qemu:///system"}, 
-                shell=True)
+                "virsh domiflist %s" % vmname,
+                env={"LIBVIRT_DEFAULT_URI": "qemu:///system"},
+                shell=True).decode('utf-8')
     except subprocess.CalledProcessError:
         module.fail_json(msg="failed to iflist dom %s" % vmname)
 
     mgmt_port = None
     fp_ports = []
 
     for l in output.split('\n'):
-        fds = re.split('\s+', l)
+        fds = re.split('\s+', l.lstrip())
         if len(fds) != 5:
             continue
         if fds[1] == "ethernet":

ansible/roles/vm_set/library/sonic_kickstart.py

@@ -42,7 +42,7 @@ def __init__(self, port, debug):
         self.d = debug
         self.d.debug('Starting')
         self.tn = Telnet('127.0.0.1', port)
-        self.tn.write('\r\n')
+        self.tn.write(b"\r\n")
 
         return
 
@@ -62,9 +62,9 @@ def cleanup(self):
     def pair(self, action, wait_for, timeout=60):
         self.d.debug('output: %s' % action)
         self.d.debug('match: %s' % ",".join(wait_for))
-        self.tn.write("%s\n" % action)
+        self.tn.write(b"%s\n" % action.encode('ascii'))
         if wait_for is not None:
-            index, match, text = self.tn.expect(wait_for, timeout)
+            index, match, text = self.tn.expect([ x.encode('ascii') for x in wait_for ], timeout)
             self.d.debug('Result of matching: %d %s %s' % (index, str(match), text))
             if index == -1:
                 raise EMatchNotFound
@@ -154,7 +154,7 @@ def main():
         result = {'kickstart_code': -1, 'changed': False, 'msg': 'EOF during the chat'}
     except EMatchNotFound:
         result = {'kickstart_code': -1, 'changed': False, 'msg': "Match for output isn't found"}
-    except Exception, e:
+    except Exception as e:
         module.fail_json(msg=str(e))
 
     module.exit_json(**result)

ansible/roles/vm_set/library/vm_topology.py

@@ -128,15 +128,15 @@ def init(self, vm_set_name, topo, vm_base, dut_fp_ports, ptf_exists=True):
                 self.vm_base_index = self.vm_names.index(vm_base)
             else:
                 raise Exception('VM_base "%s" should be presented in current vm_names: %s' % (vm_base, str(self.vm_names)))
-            for k, v in topo['VMs'].iteritems():
+            for k, v in topo['VMs'].items():
                 if self.vm_base_index + v['vm_offset'] < len(self.vm_names):
                     self.VMs[k] = v
 
-            for hostname, attrs in self.VMs.iteritems():
+            for hostname, attrs in self.VMs.items():
                 vmname = self.vm_names[self.vm_base_index + attrs['vm_offset']]
                 if len(attrs['vlans']) > len(self.get_bridges(vmname)):
                     raise Exception("Wrong vlans parameter for hostname %s, vm %s. Too many vlans. Maximum is %d" % (hostname, vmname, len(self.get_bridges(vmname))))
-            
+
         if 'host_interfaces' in topo:
             self.host_interfaces = topo['host_interfaces']
         else:
@@ -180,14 +180,14 @@ def update(self):
 
     def extract_vm_vlans(self):
         vlans = []
-        for attr in self.VMs.itervalues():
+        for attr in self.VMs.values():
             vlans.extend(attr['vlans'])
 
         return vlans
 
     def create_bridges(self):
         for vm in self.vm_names:
-            for fp_num in xrange(self.max_fp_num):
+            for fp_num in range(self.max_fp_num):
                 fp_br_name = OVS_FP_BRIDGE_TEMPLATE % (vm, fp_num)
                 self.create_ovs_bridge(fp_br_name, self.fp_mtu)
 
@@ -358,7 +358,7 @@ def unbind_mgmt_port(self, mgmt_port):
         return
 
     def bind_fp_ports(self, disconnect_vm=False):
-        for attr in self.VMs.itervalues():
+        for attr in self.VMs.values():
             for vlan_num, vlan in enumerate(attr['vlans']):
                 injected_iface = INJECTED_INTERFACES_TEMPLATE % (self.vm_set_name, vlan)
                 br_name = OVS_FP_BRIDGE_TEMPLATE % (self.vm_names[self.vm_base_index + attr['vm_offset']], vlan_num)
@@ -368,7 +368,7 @@ def bind_fp_ports(self, disconnect_vm=False):
         return
 
     def unbind_fp_ports(self):
-        for attr in self.VMs.itervalues():
+        for attr in self.VMs.values():
             for vlan_num, vlan in enumerate(attr['vlans']):
                 br_name = OVS_FP_BRIDGE_TEMPLATE % (self.vm_names[self.vm_base_index + attr['vm_offset']], vlan_num)
                 vm_iface = OVS_FP_TAP_TEMPLATE % (self.vm_names[self.vm_base_index + attr['vm_offset']], vlan_num)
@@ -385,7 +385,7 @@ def bind_vm_backplane(self):
 
         self.update()
 
-        for attr in self.VMs.itervalues():
+        for attr in self.VMs.values():
             vm_name = self.vm_names[self.vm_base_index + attr['vm_offset']]
             bp_port_name = OVS_BP_TAP_TEMPLATE % vm_name
 
@@ -517,7 +517,7 @@ def cmd(cmdline):
 
         with open(cmd_debug_fname, 'a') as fp:
             pprint("OUTPUT: %s" % stdout, fp)
-        return stdout
+        return stdout.decode('utf-8')
 
     @staticmethod
     def get_ovs_br_ports(bridge):
@@ -556,7 +556,7 @@ def get_ovs_port_bindings(bridge, vlan_iface = None):
             if vlan_iface is None or vlan_iface in result:
                 return result
             time.sleep(2*retries+1)
-        # Flow reaches here when vlan_iface not present in result 
+        # Flow reaches here when vlan_iface not present in result
         raise Exception("Can't find vlan_iface_id")
 
     @staticmethod
@@ -637,7 +637,7 @@ def check_topo(topo):
         if not isinstance(VMs, dict):
             raise Exception("topo['VMs'] should be a dictionary")
 
-        for hostname, attrs in VMs.iteritems():
+        for hostname, attrs in VMs.items():
             if 'vlans' not in attrs or not isinstance(attrs['vlans'], list):
                 raise Exception("topo['VMs']['%s'] should contain 'vlans' with a list of vlans" % hostname)
 

ansible/roles/vm_set/tasks/docker.yml

@@ -24,6 +24,13 @@
   become: yes
   when: host_distribution_version.stdout == "18.04"
 
+- name: Add docker repository for 20.04
+  apt_repository:
+    repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable
+    state: present
+  become: yes
+  when: host_distribution_version.stdout == "20.04"
+
 - name: Install docker-ce
   apt: pkg=docker-ce update_cache=yes
   become: yes
@@ -32,7 +39,12 @@
 - name: Get default pip_executable
   set_fact:
     pip_executable: pip
-  when: pip_executable is not defined
+  when: pip_executable is not defined and host_distribution_version.stdout != "20.04"
+
+- name: Get default pip_executable
+  set_fact:
+    pip_executable: pip3
+  when: pip_executable is not defined and host_distribution_version.stdout == "20.04"
 
 - name: remove old python packages
   pip: name=docker-py state=absent executable={{ pip_executable }}

ansible/roles/vm_set/tasks/main.yml

@@ -59,21 +59,38 @@
   become: yes
   with_items:
     - ifupdown
-    - python
     - qemu
     - openvswitch-switch
     - net-tools
     - bridge-utils
     - util-linux
     - iproute2
     - vlan
-    - libvirt-bin
-    - python-libvirt
-    - python-pip
     - apt-transport-https
     - ca-certificates
     - curl
     - software-properties-common
+    - libvirt-clients
+
+- name:
+  apt:
+    pkg:
+    - python
+    - libvirt-bin
+    - python-libvirt
+    - python-pip
+  become: yes
+  when: host_distribution_version.stdout == "18.04"
+
+- name:
+  apt:
+    pkg:
+    - python3-libvirt
+    - python3-pip
+    - libvirt-daemon-system
+    - qemu-system-x86
+  become: yes
+  when: host_distribution_version.stdout == "20.04"
 
 - include_tasks: docker.yml
 

ansible/roles/vm_set/templates/sonic.xml.j2

@@ -7,7 +7,7 @@
     <partition>/machine</partition>
   </resource>
   <os>
-    <type arch='x86_64' machine='pc-i440fx-1.5'>hvm</type>
+    <type arch='x86_64' machine='q35'>hvm</type>
     <boot dev='hd'/>
   </os>
   <features>