From e43c8a5067c87e774c35c7c2b9879700050afaef Mon Sep 17 00:00:00 2001 From: Pavel Shirshov Date: Thu, 17 Nov 2016 18:01:01 -0800 Subject: [PATCH] Testbed roles sync --- ansible/host_vars/STR-ACS-SERV-02.yml | 77 ++--- ansible/roles/vm_set/library/vm_network.py | 328 +++++++++++++++++++ ansible/roles/vm_set/tasks/add_bridge.yml | 10 - ansible/roles/vm_set/tasks/add_vlan.yml | 44 --- ansible/roles/vm_set/tasks/add_vlans.yml | 84 ----- ansible/roles/vm_set/tasks/delete_bridge.yml | 8 - ansible/roles/vm_set/tasks/delete_vlan.yml | 20 -- ansible/roles/vm_set/tasks/delete_vlans.yml | 41 --- ansible/roles/vm_set/tasks/renumber.yml | 30 +- ansible/roles/vm_set/tasks/start.yml | 83 +---- ansible/roles/vm_set/tasks/stop.yml | 79 +---- ansible/start_ptf_containers.yml | 4 + 12 files changed, 425 insertions(+), 383 deletions(-) create mode 100644 ansible/roles/vm_set/library/vm_network.py delete mode 100644 ansible/roles/vm_set/tasks/add_bridge.yml delete mode 100644 ansible/roles/vm_set/tasks/add_vlan.yml delete mode 100644 ansible/roles/vm_set/tasks/add_vlans.yml delete mode 100644 ansible/roles/vm_set/tasks/delete_bridge.yml delete mode 100644 ansible/roles/vm_set/tasks/delete_vlan.yml delete mode 100644 ansible/roles/vm_set/tasks/delete_vlans.yml diff --git a/ansible/host_vars/STR-ACS-SERV-02.yml b/ansible/host_vars/STR-ACS-SERV-02.yml index 6ec6d256815..8a13a072d15 100644 --- a/ansible/host_vars/STR-ACS-SERV-02.yml +++ b/ansible/host_vars/STR-ACS-SERV-02.yml @@ -6,17 +6,17 @@ ptf_1_enabled: true ptf_1_mgmt_ip: 10.255.0.189/24 ptf_1_vlan_base: 809 -ptf_2_enabled: true +ptf_2_enabled: false ptf_2_mgmt_ip: 10.255.0.51/24 -ptf_2_vlan_base: 777 +ptf_2_vlan_base: 0 # ??? ptf_3_enabled: true ptf_3_mgmt_ip: 10.255.0.186/24 ptf_3_vlan_base: 841 -ptf_4_enabled: false +ptf_4_enabled: true ptf_4_mgmt_ip: 10.255.0.184/24 -ptf_4_vlan_base: 457 # ??? +ptf_4_vlan_base: 969 ptf_5_enabled: false ptf_5_mgmt_ip: 10.255.0.183/24 @@ -224,8 +224,9 @@ VMs_1: vlans: "16" memory: "{{ spine_memory }}" -vm_2_enabled: false -vm_2_vlan_base: 101 # ??? + +vm_2_enabled: true +vm_2_vlan_base: 777 vm_2_injected_ptf: true vm_2_injected_ip: 10.255.0.181/24 VMs_2: @@ -233,192 +234,192 @@ VMs_2: num: 0 filename: 04-ARISTA01T0.xml serial_port: 5032 - vlans: "1,2,3,4,5,6,7,8" + vlans: "17" memory: "{{ tor_memory }}" ARISTA01T2: num: 1 filename: 04-ARISTA01T2.xml serial_port: 5048 - vlans: "9,10,11,12,13,14,15,16" + vlans: "1" memory: "{{ spine_memory }}" ARISTA02T0: num: 2 filename: 04-ARISTA02T0.xml serial_port: 5033 - vlans: "17,18,19,20,21,22,23,24" + vlans: "18" memory: "{{ tor_memory }}" ARISTA02T2: num: 3 filename: 04-ARISTA02T2.xml serial_port: 5049 - vlans: "25,26,27,28,29,30.31,32" + vlans: "2" memory: "{{ spine_memory }}" ARISTA03T0: num: 4 filename: 04-ARISTA03T0.xml serial_port: 5034 - vlans: "" + vlans: "19" memory: "{{ tor_memory }}" ARISTA03T2: num: 5 filename: 04-ARISTA03T2.xml serial_port: 5050 - vlans: "" + vlans: "3" memory: "{{ spine_memory }}" ARISTA04T0: num: 6 filename: 04-ARISTA04T0.xml serial_port: 5035 - vlans: "" + vlans: "20" memory: "{{ tor_memory }}" ARISTA04T2: num: 7 filename: 04-ARISTA04T2.xml serial_port: 5051 - vlans: "" + vlans: "4" memory: "{{ spine_memory }}" ARISTA05T0: num: 8 filename: 04-ARISTA05T0.xml serial_port: 5036 - vlans: "" + vlans: "21" memory: "{{ tor_memory }}" ARISTA05T2: num: 9 filename: 04-ARISTA05T2.xml serial_port: 5052 - vlans: "" + vlans: "5" memory: "{{ spine_memory }}" ARISTA06T0: num: 10 filename: 04-ARISTA06T0.xml serial_port: 5037 - vlans: "" + vlans: "22" memory: "{{ tor_memory }}" ARISTA06T2: num: 11 filename: 04-ARISTA06T2.xml serial_port: 5053 - vlans: "" + vlans: "6" memory: "{{ spine_memory }}" ARISTA07T0: num: 12 filename: 04-ARISTA07T0.xml serial_port: 5038 - vlans: "" + vlans: "23" memory: "{{ tor_memory }}" ARISTA07T2: num: 13 filename: 04-ARISTA07T2.xml serial_port: 5054 - vlans: "" + vlans: "7" memory: "{{ spine_memory }}" ARISTA08T0: num: 14 filename: 04-ARISTA08T0.xml serial_port: 5039 - vlans: "" + vlans: "24" memory: "{{ tor_memory }}" ARISTA08T2: num: 15 filename: 04-ARISTA08T2.xml serial_port: 5055 - vlans: "" + vlans: "8" memory: "{{ spine_memory }}" ARISTA09T0: num: 16 filename: 04-ARISTA09T0.xml serial_port: 5040 - vlans: "" + vlans: "25" memory: "{{ tor_memory }}" ARISTA09T2: num: 17 filename: 04-ARISTA09T2.xml serial_port: 5056 - vlans: "" + vlans: "9" memory: "{{ spine_memory }}" ARISTA10T0: num: 18 filename: 04-ARISTA10T0.xml serial_port: 5041 - vlans: "" + vlans: "26" memory: "{{ tor_memory }}" ARISTA10T2: num: 19 filename: 04-ARISTA10T2.xml serial_port: 5057 - vlans: "" + vlans: "10" memory: "{{ spine_memory }}" ARISTA11T0: num: 20 filename: 04-ARISTA11T0.xml serial_port: 5042 - vlans: "" + vlans: "27" memory: "{{ tor_memory }}" ARISTA11T2: num: 21 filename: 04-ARISTA11T2.xml serial_port: 5058 - vlans: "" + vlans: "11" memory: "{{ spine_memory }}" ARISTA12T0: num: 22 filename: 04-ARISTA12T0.xml serial_port: 5043 - vlans: "" + vlans: "28" memory: "{{ tor_memory }}" ARISTA12T2: num: 23 filename: 04-ARISTA12T2.xml serial_port: 5059 - vlans: "" + vlans: "12" memory: "{{ spine_memory }}" ARISTA13T0: num: 24 filename: 04-ARISTA13T0.xml serial_port: 5044 - vlans: "" + vlans: "29" memory: "{{ tor_memory }}" ARISTA13T2: num: 25 filename: 04-ARISTA13T2.xml serial_port: 5060 - vlans: "" + vlans: "13" memory: "{{ spine_memory }}" ARISTA14T0: num: 26 filename: 04-ARISTA14T0.xml serial_port: 5045 - vlans: "" + vlans: "30" memory: "{{ tor_memory }}" ARISTA14T2: num: 27 filename: 04-ARISTA14T2.xml serial_port: 5061 - vlans: "" + vlans: "14" memory: "{{ spine_memory }}" ARISTA15T0: num: 28 filename: 04-ARISTA15T0.xml serial_port: 5046 - vlans: "" + vlans: "31" memory: "{{ tor_memory }}" ARISTA15T2: num: 29 filename: 04-ARISTA15T2.xml serial_port: 5062 - vlans: "" + vlans: "15" memory: "{{ spine_memory }}" ARISTA16T0: num: 30 filename: 04-ARISTA16T0.xml serial_port: 5047 - vlans: "" + vlans: "32" memory: "{{ tor_memory }}" ARISTA16T2: num: 31 filename: 04-ARISTA16T2.xml serial_port: 5063 - vlans: "" + vlans: "16" memory: "{{ spine_memory }}" diff --git a/ansible/roles/vm_set/library/vm_network.py b/ansible/roles/vm_set/library/vm_network.py new file mode 100644 index 00000000000..253caeb1873 --- /dev/null +++ b/ansible/roles/vm_set/library/vm_network.py @@ -0,0 +1,328 @@ +#!/usr/bin/python + +import subprocess +import re +from docker import Client +from ansible.module_utils.basic import * + +DOCUMENTATION = ''' +--- +module: vm_network_create +version_added: "0.1" +author: Pavel Shirshov (pavelsh@microsoft.com) +short_description: Generate virtual network for a set of VMs +description: + - With cmd: 'create' the module: + - creates 32*8 ovs bridges with name template "br-vs{{ vm_set_id }}-vm{{ vm_set_dict[]['num']}}-{{ 0..7 }}" which will be used by FP port of VMs + - creates a linux bridge with name {{ port1_bridge }} for backplane connectivity between VMs + - With cmd: 'destroy' the module: + - destroys 32*8 ovs bridges with name template "br-vs{{ vm_set_id }}-vm{{ vm_set_dict[]['num']}}-{{ 0..7 }}" which were used by FP port of VMs + - destroys a linux bridge with name {{ port1_bridge }} for backplane connectivity between VMs + - With cmd: 'bind' the module: + - creates 32 vlan interfaces on the external interface + - bind this interfaces to the ovs bridges which were created by 'create' command + - bind corresponing interface from ptf_injected container to the ovs bridges + - With cmd: 'unbind' the module: + - destroys 32 vlan interfaces from the external interface + +Parameters: + - cmd: One of the commands: 'create', 'bind', 'unbind', 'destroy' + - vm_set_id: identifier for the VM set, a number + - port1_bridge: name of the bridge which will be created for the backplane connectivity + - vm_set_dict: dictionary with VM parameters. Check host_vars/STR-ACS-SERV-0x.yml for details + - fp_mtu: MTU for FP ports + - ext_iface: physical interface which will be used for for vlan creation + - vlan_base: the first vlan for the network +''' + +EXAMPLES = ''' +- name: Create VM set network. vm set {{ id }} + vm_network: + cmd: 'create' + vm_set_id: "{{ id }}" + port1_bridge: "{{ port1_bridge }}" + vm_set_dict: "{{ VMs }}" + fp_mtu: "{{ fp_mtu_size }}" + ext_iface: "{{ external_iface }}" + vlan_base: "{{ vlan_base }}" +''' + +DEFAULT_MTU = 0 +NUM_FP_VLANS_PER_FP = 8 +OVS_BRIDGE_TEMPLATE = 'br-vs%d-vm%d-%d' +INJECTED_INTERFACES_TEMPLATE = "inje-%d-%d" + +class VMNetwork(object): + + def __init__(self, vm_set_id, port1_bridge, vm_set_dict, ext_iface, vlan_base, fp_mtu=DEFAULT_MTU): + self.vm_set_id = vm_set_id + self.port1_bridge = port1_bridge + self.vm_set_dict = vm_set_dict + self.ext_iface = ext_iface + self.vlan_base = vlan_base + self.fp_mtu = fp_mtu + + self.host_ifaces = VMNetwork.ifconfig('ifconfig -a') + + return + + def create_port1_bridge(self): + if self.port1_bridge not in self.host_ifaces: + VMNetwork.cmd('brctl addbr %s' % self.port1_bridge) + + VMNetwork.cmd('ifconfig %s up' % self.port1_bridge) + + return + + def destroy_port1_bridge(self): + if self.port1_bridge in self.host_ifaces: + VMNetwork.cmd('ifconfig %s down' % self.port1_bridge) + VMNetwork.cmd('brctl delbr %s' % self.port1_bridge) + + return + + def create_fp_bridges(self): + for vm in self.vm_set_dict.itervalues(): + for vlan_num in xrange(NUM_FP_VLANS_PER_FP): + self.create_fp_bridge(vm["num"], vlan_num) + + return + + def create_fp_bridge(self, vm_num, vlan_num): + vlan_name = OVS_BRIDGE_TEMPLATE % (self.vm_set_id, int(vm_num), vlan_num) + + if vlan_name not in self.host_ifaces: + VMNetwork.cmd('ovs-vsctl add-br %s' % vlan_name) + + if self.fp_mtu != DEFAULT_MTU: + VMNetwork.cmd('ifconfig %s mtu %d' % (vlan_name, self.fp_mtu)) + + VMNetwork.cmd('ifconfig %s up' % vlan_name) + + return + + def destroy_fp_bridges(self): + for vm in self.vm_set_dict.itervalues(): + for vlan_num in xrange(NUM_FP_VLANS_PER_FP): + self.destroy_fp_bridge(vm["num"], vlan_num) + + return + + + def destroy_fp_bridge(self, vm_num, vlan_num): + vlan_name = OVS_BRIDGE_TEMPLATE % (self.vm_set_id, int(vm_num), vlan_num) + + if vlan_name in self.host_ifaces: + VMNetwork.cmd('ifconfig %s down' % vlan_name) + VMNetwork.cmd('ovs-vsctl del-br %s' % vlan_name) + + return + + def up_ext_iface(self): + if self.ext_iface in self.host_ifaces: + VMNetwork.cmd('ifconfig %s up' % self.ext_iface) + + return + + def check_vlans(self, vlans_str, vlans): + if len(vlans) == 0: + return + + if len(vlans) > 8: + raise Exception("Wrong vlans parameter. Too many vlans. Maximum is 8: %s" % vlans_str) + + for vlan in vlans_str.split(','): + if not vlan.isdigit(): + raise Exception("Wrong vlans parameter: %s" % vlans_str) + + for vlan in vlans: + if int(vlan) > 32: # FIXME: -1 + raise Exception("Vlan offset %s supposed to be not more then 32: %s" % (vlan, vlans_str)) + + return + + def bind(self): + for vm in self.vm_set_dict.itervalues(): + vm_num = vm['num'] + vlans_str = vm['vlans'] + vlans = [int(vlan) for vlan in vlans_str.split(',')] + self.check_vlans(vlans_str, vlans) + for vlan_num, vlan in enumerate(vlans): + vlan_id = self.vlan_base + vlan - 1 # FIXME: finally change the VMs in host vars with right offsets + vlan_iface = "%s.%d" % (self.ext_iface, vlan_id) + injected_iface = INJECTED_INTERFACES_TEMPLATE % (self.vm_set_id, (vlan - 1)) # FIXME: this -1 thing + port0_bridge = OVS_BRIDGE_TEMPLATE % (self.vm_set_id, int(vm_num), vlan_num) + self.create_phys_vlan(vlan_iface, vlan_id) + self.bind_phys_vlan(port0_bridge, vlan_iface, injected_iface) + + return + + def create_phys_vlan(self, vlan_iface, vlan_id): + if vlan_iface not in self.host_ifaces: + VMNetwork.cmd('vconfig add %s %d' % (self.ext_iface, vlan_id)) + + VMNetwork.cmd('ifconfig %s up' % vlan_iface) + + return + + def bind_phys_vlan(self, br_name, vlan_iface, injected_iface): + ports = VMNetwork.get_ovs_br_ports(br_name) + + if injected_iface not in ports: + VMNetwork.cmd('ovs-vsctl add-port %s %s' % (br_name, injected_iface)) + + if vlan_iface not in ports: + VMNetwork.cmd('ovs-vsctl add-port %s %s' % (br_name, vlan_iface)) + + bindings = VMNetwork.get_ovs_port_bindings(br_name) + vlan_iface_id = bindings[vlan_iface] + + # clear old bindings + VMNetwork.cmd('ovs-ofctl del-flows %s' % br_name) + + # Add flow from a VM to an external iface + VMNetwork.cmd("ovs-ofctl add-flow %s table=0,in_port=1,action=output:%s" % (br_name, vlan_iface_id)) + + # Add flow from external iface to a VM and a ptf container + VMNetwork.cmd("ovs-ofctl add-flow %s table=0,in_port=%s,action=output:1,2" % (br_name, vlan_iface_id)) + + # Add flow from a ptf container to an external iface + VMNetwork.cmd("ovs-ofctl add-flow %s table=0,in_port=2,action=output:%s" % (br_name, vlan_iface_id)) + + return + + def unbind(self): + # try vlans from the host_vars + for vm in self.vm_set_dict.itervalues(): + vm_num = vm['num'] + vlans_str = vm['vlans'] + vlans = [int(vlan) for vlan in vlans_str.split(',')] + self.check_vlans(vlans_str, vlans) + for vlan_num, vlan in enumerate(vlans): + vlan_id = self.vlan_base + vlan - 1 # FIXME: finally change the VMs in host vars with right offsets + vlan_iface = "%s.%d" % (self.ext_iface, vlan_id) + injected_iface = INJECTED_INTERFACES_TEMPLATE % (self.vm_set_id, (vlan - 1)) # FIXME: fix this -1 thing + port0_bridge = OVS_BRIDGE_TEMPLATE % (self.vm_set_id, int(vm_num), vlan_num) + self.unbind_phys_vlan(port0_bridge, vlan_iface) + self.destroy_phys_vlan(vlan_iface) + + # try vlans from the ovs db + for vm in self.vm_set_dict.itervalues(): + vm_num = vm['num'] + for vlan_num in xrange(NUM_FP_VLANS_PER_FP): + bridge_name = OVS_BRIDGE_TEMPLATE % (self.vm_set_id, int(vm_num), vlan_num) + ports = VMNetwork.get_ovs_port_bindings(bridge_name) + for port in ports.iterkeys(): + if self.ext_iface in port: + self.unbind_phys_vlan(bridge_name, port) + self.destroy_phys_vlan(port) + + return + + def destroy_phys_vlan(self, vlan_iface): + if vlan_iface in self.host_ifaces: + VMNetwork.cmd('ifconfig %s down' % vlan_iface) + VMNetwork.cmd('vconfig rem %s' % vlan_iface) + + return + + def unbind_phys_vlan(self, br_name, vlan_iface): + ports = VMNetwork.get_ovs_br_ports(br_name) + + if vlan_iface in ports: + VMNetwork.cmd('ovs-vsctl del-port %s %s' % (br_name, vlan_iface)) + + return + + @staticmethod + def cmd(cmdline): + cmd = cmdline.split(' ') + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + ret_code = process.returncode + + if ret_code != 0: + raise Exception("ret_code=%d, error message=%s. cmd=%s" % (ret_code, stderr, cmdline)) + + return stdout + + @staticmethod + def get_ovs_br_ports(bridge): + out = VMNetwork.cmd('ovs-vsctl list-ports %s' % bridge) + return set(out.split('\n')) + + @staticmethod + def get_ovs_port_bindings(bridge): + out = VMNetwork.cmd('ovs-ofctl show %s' % bridge) + lines = out.split('\n') + result = {} + for line in lines: + matched = re.match(r'^\s+(\S+)\((\S+)\):\s+addr:.+$', line) + if matched: + port_id = matched.group(1) + iface_name = matched.group(2) + result[iface_name] = port_id + + return result + + @staticmethod + def ifconfig(cmdline): + out = VMNetwork.cmd(cmdline) + + ifaces = set() + + rows = out.split('\n') + for row in rows: + if len(row) == 0: + continue + terms = row.split() + if not row[0].isspace(): + ifaces.add(terms[0]) + + return ifaces + + +def main(): + module = AnsibleModule( + argument_spec=dict( + cmd=dict(required=True, choices=['create', 'bind', 'unbind', 'destroy']), + vm_set_id=dict(required=True, type='int'), + port1_bridge=dict(required=True, type='str'), + vm_set_dict=dict(required=True, type='dict'), + fp_mtu=dict(required=False, type='int', default=DEFAULT_MTU), + ext_iface=dict(required=True, type='str'), + vlan_base=dict(required=True, type='int')), + supports_check_mode=False) + + cmd = module.params['cmd'] + vm_set_id = module.params['vm_set_id'] + port1_bridge = module.params['port1_bridge'] + vm_set_dict = module.params['vm_set_dict'] + fp_mtu = module.params['fp_mtu'] + ext_iface = module.params['ext_iface'] + vlan_base = module.params['vlan_base'] + + try: + net = VMNetwork(vm_set_id, port1_bridge, vm_set_dict, ext_iface, vlan_base, fp_mtu) + if cmd == 'create': + net.create_port1_bridge() + net.create_fp_bridges() + elif cmd == 'destroy': + net.destroy_port1_bridge() + net.destroy_fp_bridges() + elif cmd == 'bind': + net.up_ext_iface() + net.bind() + elif cmd == 'unbind': + net.unbind() + else: + raise Exception("Got wrong cmd: %s. Ansible bug?" % cmd) + + except Exception as error: + module.fail_json(msg=str(error)) + + module.exit_json(changed=True) + +if __name__ == "__main__": + main() + diff --git a/ansible/roles/vm_set/tasks/add_bridge.yml b/ansible/roles/vm_set/tasks/add_bridge.yml deleted file mode 100644 index 75dc4e85ed6..00000000000 --- a/ansible/roles/vm_set/tasks/add_bridge.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Create port0 bridge {{ port0_bridge }}. vm set {{ id }} - command: ovs-vsctl add-br {{ port0_bridge }} - when: port0_bridge not in ansible_interfaces - -- name: Configure MTU of the bridge {{ port0_bridge }} MTU={{ fp_mtu_size }}. vm set {{ id }} - command: ifconfig {{ port0_bridge }} mtu {{ fp_mtu_size }} - -- name: Enable port0 bridge {{ port0_bridge }}. vm set {{ id }} - command: ifconfig {{ port0_bridge }} up - diff --git a/ansible/roles/vm_set/tasks/add_vlan.yml b/ansible/roles/vm_set/tasks/add_vlan.yml deleted file mode 100644 index 0cd6391f939..00000000000 --- a/ansible/roles/vm_set/tasks/add_vlan.yml +++ /dev/null @@ -1,44 +0,0 @@ -- name: Create the external vlan {{ external_iface }} {{ vlan }}. vm set {{ id }} - command: vconfig add {{ external_iface }} {{ vlan }} - when: vlan_iface not in ansible_interfaces - -- name: Enable the external vlan {{ vlan_iface }}. vm set {{ id }} - command: ifconfig {{ vlan_iface }} up - -- name: Get list-ports output {{ port0_bridge }}. vm set {{ id }} - command: ovs-vsctl list-ports {{ port0_bridge }} - register: cur_bridges - -- name: Connect port0 to the external vlan {{ port0_bridge }} {{ vlan_iface }}. vm set {{ id }} - command: ovs-vsctl add-port {{ port0_bridge }} {{ vlan_iface }} - when: cur_bridges.stdout.find( vlan_iface ) == -1 - -- name: Connect port0 to injected port {{ port0_bridge }} {{ injected_iface }}. vm set {{ id }} - command: ovs-vsctl add-port {{ port0_bridge }} {{ injected_iface }} - when: cur_bridges.stdout.find( injected_iface ) == -1 and vm_injected_ptf - -- name: Define port number for vlan interface {{ vlan_iface }} in {{ port0_bridge }}. vm set {{ id }} - shell: ovs-ofctl show {{ port0_bridge }} | sed -n '/(\(.*\..*\)).*addr/ s/ \(.*\)(.*/\1/p' - register: port_number_digits - -- name: Setting ext_port_number={{ port_number_digits.stdout }}. vm set {{ id }} - set_fact: ext_port_number={{ port_number_digits.stdout|int }} - -- name: Remove standard rules from the ovs bridge {{ port0_bridge }}. vm set {{ id }} - command: ovs-ofctl del-flows {{ port0_bridge }} - -- name: Add first forwarding rule for the ovs bridge {{ port0_bridge }}. vm set {{ id }} - command: ovs-ofctl add-flow {{ port0_bridge }} 'table=0,in_port=1,action=output:{{ ext_port_number }}' - -- name: Add second forwarding rule for the ovs bridge {{ port0_bridge }}. vm set {{ id }} - command: ovs-ofctl add-flow {{ port0_bridge }} 'table=0,in_port={{ ext_port_number }},action=output:1' - when: not vm_injected_ptf - -- name: Add second forwarding rule for the ovs bridge {{ port0_bridge }}. vm set {{ id }} - command: ovs-ofctl add-flow {{ port0_bridge }} 'table=0,in_port={{ ext_port_number }},actions=3,1' - when: vm_injected_ptf - -- name: Add third forwarding rule for the ovs bridge {{ port0_bridge }}. vm set {{ id }} - command: ovs-ofctl add-flow {{ port0_bridge }} 'table=0,in_port=3,actions={{ ext_port_number }}' - when: vm_injected_ptf - diff --git a/ansible/roles/vm_set/tasks/add_vlans.yml b/ansible/roles/vm_set/tasks/add_vlans.yml deleted file mode 100644 index c645615dc55..00000000000 --- a/ansible/roles/vm_set/tasks/add_vlans.yml +++ /dev/null @@ -1,84 +0,0 @@ -- name: Expand vlan list - set_fact: vlan_arr={{ vlans.split(',') + ['-1', '-1', '-1', '-1', '-1', '-1', '-1', '-1'] }} - -# FIXME: my ansible doesn't support nested loops. So below are copy of everything -- name: Add external interfaces. Vlan 0. vm set {{ id }} - include: add_vlan.yml - vars: - vlan_id: "{{ vlan_arr[0]|int - 1}}" - port0_bridge: "br-vs{{ id }}-vm{{ num }}-0" - vlan: "{{ vlan_base|int + vlan_id|int }}" - vlan_iface: "{{ external_iface }}.{{ vlan_base|int + vlan_id|int }}" - injected_iface: "inje-{{ id }}-{{ vlan_id }}" - when: vlan_arr[0] != '-1' - -- name: Add external interfaces. Vlan 1. vm set {{ id }} - include: add_vlan.yml - vars: - vlan_id: "{{ vlan_arr[1]|int - 1}}" - port0_bridge: "br-vs{{ id }}-vm{{ num }}-1" - vlan: "{{ vlan_base|int + vlan_id|int }}" - vlan_iface: "{{ external_iface }}.{{ vlan_base|int + vlan_id|int }}" - injected_iface: "inje-{{ id }}-{{ vlan_id }}" - when: vlan_arr[1] != '-1' - -- name: Add external interfaces. Vlan 2. vm set {{ id }} - include: add_vlan.yml - vars: - vlan_id: "{{ vlan_arr[2]|int - 1}}" - port0_bridge: "br-vs{{ id }}-vm{{ num }}-2" - vlan: "{{ vlan_base|int + vlan_id|int }}" - vlan_iface: "{{ external_iface }}.{{ vlan_base|int + vlan_id|int }}" - injected_iface: "inje-{{ id }}-{{ vlan_id }}" - when: vlan_arr[2] != '-1' - -- name: Add external interfaces. Vlan 3. vm set {{ id }} - include: add_vlan.yml - vars: - vlan_id: "{{ vlan_arr[3]|int - 1}}" - port0_bridge: "br-vs{{ id }}-vm{{ num }}-3" - vlan: "{{ vlan_base|int + vlan_id|int }}" - vlan_iface: "{{ external_iface }}.{{ vlan_base|int + vlan_id|int }}" - injected_iface: "inje-{{ id }}-{{ vlan_id }}" - when: vlan_arr[3] != '-1' - -- name: Add external interfaces. Vlan 4. vm set {{ id }} - include: add_vlan.yml - vars: - vlan_id: "{{ vlan_arr[4]|int - 1}}" - port0_bridge: "br-vs{{ id }}-vm{{ num }}-4" - vlan: "{{ vlan_base|int + vlan_id|int }}" - vlan_iface: "{{ external_iface }}.{{ vlan_base|int + vlan_id|int }}" - injected_iface: "inje-{{ id }}-{{ vlan_id }}" - when: vlan_arr[4] != '-1' - -- name: Add external interfaces. Vlan 5. vm set {{ id }} - include: add_vlan.yml - vars: - vlan_id: "{{ vlan_arr[5]|int - 1}}" - port0_bridge: "br-vs{{ id }}-vm{{ num }}-5" - vlan: "{{ vlan_base|int + vlan_id|int }}" - vlan_iface: "{{ external_iface }}.{{ vlan_base|int + vlan_id|int }}" - injected_iface: "inje-{{ id }}-{{ vlan_id }}" - when: vlan_arr[5] != '-1' - -- name: Add external interfaces. Vlan 6. vm set {{ id }} - include: add_vlan.yml - vars: - vlan_id: "{{ vlan_arr[6]|int - 1}}" - port0_bridge: "br-vs{{ id }}-vm{{ num }}-6" - vlan: "{{ vlan_base|int + vlan_id|int }}" - vlan_iface: "{{ external_iface }}.{{ vlan_base|int + vlan_id|int }}" - injected_iface: "inje-{{ id }}-{{ vlan_id }}" - when: vlan_arr[6] != '-1' - -- name: Add external interfaces. Vlan 7. vm set {{ id }} - include: add_vlan.yml - vars: - vlan_id: "{{ vlan_arr[7]|int - 1}}" - port0_bridge: "br-vs{{ id }}-vm{{ num }}-7" - vlan: "{{ vlan_base|int + vlan_id|int }}" - vlan_iface: "{{ external_iface }}.{{ vlan_base|int + vlan_id|int }}" - injected_iface: "inje-{{ id }}-{{ vlan_id }}" - when: vlan_arr[7] != '-1' - diff --git a/ansible/roles/vm_set/tasks/delete_bridge.yml b/ansible/roles/vm_set/tasks/delete_bridge.yml deleted file mode 100644 index cff75ce7e05..00000000000 --- a/ansible/roles/vm_set/tasks/delete_bridge.yml +++ /dev/null @@ -1,8 +0,0 @@ -- name: Disable port0 vm bridge {{ port0_bridge }}. vm set {{ id }} - command: ifconfig {{ port0_bridge }} down - when: port0_bridge in ansible_interfaces - -- name: Destroy port0 vm bridge {{ port0_bridge }}. vm set {{ id }} - command: ovs-vsctl del-br {{ port0_bridge }} - when: port0_bridge in ansible_interfaces - diff --git a/ansible/roles/vm_set/tasks/delete_vlan.yml b/ansible/roles/vm_set/tasks/delete_vlan.yml deleted file mode 100644 index 68e28fc9727..00000000000 --- a/ansible/roles/vm_set/tasks/delete_vlan.yml +++ /dev/null @@ -1,20 +0,0 @@ -- name: Find name of vlan iface {{ port0_bridge }}. vm set {{ id }} - shell: ovs-vsctl list-ports {{ port0_bridge }} | grep {{ external_iface }} - register: vlan_out - failed_when: False - -- name: Extract vlan name {{ port0_bridge }}. vm set {{ id }} - set_fact: vlan_iface={{ vlan_out.stdout|default('not_exist') }} - -- name: Remove portinterface {{ vlan_iface }} from the bridge {{ port0_bridge }}. vm set {{ id }} - command: ovs-vsctl del-port {{ port0_bridge }} {{ vlan_iface }} - when: vlan_out.rc == 0 - -- name: Disable external vlan {{ vlan_iface }}. vm set {{ id }} - command: ifconfig {{ vlan_iface }} down - when: vlan_iface in ansible_interfaces - -- name: Destroy external vlan {{ vlan_iface }}. vm set {{ id }} - command: vconfig rem {{ vlan_iface }} - when: vlan_iface in ansible_interfaces - diff --git a/ansible/roles/vm_set/tasks/delete_vlans.yml b/ansible/roles/vm_set/tasks/delete_vlans.yml deleted file mode 100644 index 268ef4e2687..00000000000 --- a/ansible/roles/vm_set/tasks/delete_vlans.yml +++ /dev/null @@ -1,41 +0,0 @@ -# FIXME: my ansible doesn't support nested loops. So below are copy of everything -- name: Remove external interface. Vlan 0. vm set {{ id }} - include: delete_vlan.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ num }}-0" - -- name: Remove external interface. Vlan 1. vm set {{ id }} - include: delete_vlan.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ num }}-1" - -- name: Remove external interface. Vlan 2. vm set {{ id }} - include: delete_vlan.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ num }}-2" - -- name: Remove external interface. Vlan 3. vm set {{ id }} - include: delete_vlan.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ num }}-3" - -- name: Remove external interface. Vlan 4. vm set {{ id }} - include: delete_vlan.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ num }}-4" - -- name: Remove external interface. Vlan 5. vm set {{ id }} - include: delete_vlan.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ num }}-5" - -- name: Remove external interface. Vlan 6. vm set {{ id }} - include: delete_vlan.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ num }}-6" - -- name: Remove external interface. Vlan 7. vm set {{ id }} - include: delete_vlan.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ num }}-7" - diff --git a/ansible/roles/vm_set/tasks/renumber.yml b/ansible/roles/vm_set/tasks/renumber.yml index bebc535e8e4..b5a03537e9d 100644 --- a/ansible/roles/vm_set/tasks/renumber.yml +++ b/ansible/roles/vm_set/tasks/renumber.yml @@ -1,14 +1,20 @@ -- name: Remove external interfaces. vm set {{ id }} - include: delete_vlans.yml - vars: - num: "{{ item.value.num }}" - with_dict: "{{ VMs }}" +- name: Unbind external ifaces to VM set. vm set {{ id }} + vm_network: + cmd: 'unbind' + vm_set_id: "{{ id }}" + port1_bridge: "{{ port1_bridge }}" + vm_set_dict: "{{ VMs }}" + fp_mtu: "{{ fp_mtu_size }}" + ext_iface: "{{ external_iface }}" + vlan_base: "{{ vlan_base }}" -- name: Add external interfaces. vm set {{ id }} - include: add_vlans.yml - vars: - num: "{{ item.value.num }}" - vlans: "{{ item.value.vlans }}" - when: vlans != "" - with_dict: "{{ VMs }}" +- name: Bind external ifaces to VM set. vm set {{ id }} + vm_network: + cmd: 'bind' + vm_set_id: "{{ id }}" + port1_bridge: "{{ port1_bridge }}" + vm_set_dict: "{{ VMs }}" + fp_mtu: "{{ fp_mtu_size }}" + ext_iface: "{{ external_iface }}" + vlan_base: "{{ vlan_base }}" diff --git a/ansible/roles/vm_set/tasks/start.yml b/ansible/roles/vm_set/tasks/start.yml index 92881008788..2e815a83683 100644 --- a/ansible/roles/vm_set/tasks/start.yml +++ b/ansible/roles/vm_set/tasks/start.yml @@ -45,61 +45,15 @@ mgmt_bridge: "{{ mgmt_bridge }}" when: vm_injected_ptf -- name: Create port1 bridge {{ port1_bridge }}. vm set {{ id }} - command: brctl addbr {{ port1_bridge }} - when: port1_bridge not in ansible_interfaces - -- name: Enable port1 bridge {{ port1_bridge }}. vm set {{ id }} - command: ifconfig {{ port1_bridge }} up - -# FIXME: my ansible doesn't support nested loops. So below are copy of everything -- name: Add bridges for fp. vm set {{ id }}. Vlan 0 - include: add_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-0" - with_dict: "{{ VMs }}" - -- name: Add bridges for fp. vm set {{ id }}. Vlan 1 - include: add_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-1" - with_dict: "{{ VMs }}" - -- name: Add bridges for fp. vm set {{ id }}. Vlan 2 - include: add_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-2" - with_dict: "{{ VMs }}" - -- name: Add bridges for fp. vm set {{ id }}. Vlan 3 - include: add_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-3" - with_dict: "{{ VMs }}" - -- name: Add bridges for fp. vm set {{ id }}. Vlan 4 - include: add_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-4" - with_dict: "{{ VMs }}" - -- name: Add bridges for fp. vm set {{ id }}. Vlan 5 - include: add_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-5" - with_dict: "{{ VMs }}" - -- name: Add bridges for fp. vm set {{ id }}. Vlan 6 - include: add_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-6" - with_dict: "{{ VMs }}" - -- name: Add bridges for fp. vm set {{ id }}. Vlan 7 - include: add_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-7" - with_dict: "{{ VMs }}" +- name: Create VM set network. vm set {{ id }} + vm_network: + cmd: 'create' + vm_set_id: "{{ id }}" + port1_bridge: "{{ port1_bridge }}" + vm_set_dict: "{{ VMs }}" + fp_mtu: "{{ fp_mtu_size }}" + ext_iface: "{{ external_iface }}" + vlan_base: "{{ vlan_base }}" - name: Start VMs. vm set {{ id }} include: start_vm.yml @@ -130,14 +84,13 @@ port1_tap: "{{ vm_name }}-p1" with_dict: "{{ VMs }}" -- name: Enable external interface {{ external_iface }}. vm set {{ id }} - command: ifconfig {{ external_iface }} up - -- name: Add external interfaces. vm set {{ id }} - include: add_vlans.yml - vars: - num: "{{ item.value.num }}" - vlans: "{{ item.value.vlans }}" - when: vlans != "" - with_dict: "{{ VMs }}" +- name: Bind external ifaces to VM set. vm set {{ id }} + vm_network: + cmd: 'bind' + vm_set_id: "{{ id }}" + port1_bridge: "{{ port1_bridge }}" + vm_set_dict: "{{ VMs }}" + fp_mtu: "{{ fp_mtu_size }}" + ext_iface: "{{ external_iface }}" + vlan_base: "{{ vlan_base }}" diff --git a/ansible/roles/vm_set/tasks/stop.yml b/ansible/roles/vm_set/tasks/stop.yml index 1ee1138415e..a3fe828c287 100644 --- a/ansible/roles/vm_set/tasks/stop.yml +++ b/ansible/roles/vm_set/tasks/stop.yml @@ -5,11 +5,15 @@ disk_image: "{{ root_path }}/disks/{{ vm_name }}_hdd.vmdk" with_dict: "{{ VMs }}" -- name: Remove external interfaces. vm set {{ id }} - include: delete_vlans.yml - vars: - num: "{{ item.value.num }}" - with_dict: "{{ VMs }}" +- name: Unbind external ifaces to VM set. vm set {{ id }} + vm_network: + cmd: 'unbind' + vm_set_id: "{{ id }}" + port1_bridge: "{{ port1_bridge }}" + vm_set_dict: "{{ VMs }}" + fp_mtu: "{{ fp_mtu_size }}" + ext_iface: "{{ external_iface }}" + vlan_base: "{{ vlan_base }}" - name: Remove ptf docker container. vm set {{ id }} docker: @@ -17,60 +21,13 @@ image: "{{ docker_registry_host }}/{{ docker_image }}" state: absent -# FIXME: my ansible doesn't support nested loops. So below are copy of everything -- name: Remove bridges. vlan 0. vm set {{ id }} - include: delete_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-0" - with_dict: "{{ VMs }}" - -- name: Remove bridges. vlan 1. vm set {{ id }} - include: delete_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-1" - with_dict: "{{ VMs }}" - -- name: Remove bridges. vlan 2. vm set {{ id }} - include: delete_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-2" - with_dict: "{{ VMs }}" - -- name: Remove bridges. vlan 3. vm set {{ id }} - include: delete_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-3" - with_dict: "{{ VMs }}" - -- name: Remove bridges. vlan 4. vm set {{ id }} - include: delete_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-4" - with_dict: "{{ VMs }}" - -- name: Remove bridges. vlan 5. vm set {{ id }} - include: delete_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-5" - with_dict: "{{ VMs }}" - -- name: Remove bridges. vlan 6. vm set {{ id }} - include: delete_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-6" - with_dict: "{{ VMs }}" - -- name: Remove bridges. vlan 7. vm set {{ id }} - include: delete_bridge.yml - vars: - port0_bridge: "br-vs{{ id }}-vm{{ item.value.num }}-7" - with_dict: "{{ VMs }}" - -- name: Disable port1 bridge {{ port1_bridge }}. vm set {{ id }} - command: ifconfig {{ port1_bridge }} down - when: port1_bridge in ansible_interfaces - -- name: Destroy port1 bridge {{ port1_bridge }}. vm set {{ id }} - command: brctl delbr {{ port1_bridge }} - when: port1_bridge in ansible_interfaces +- name: Destroy VM set network. vm set {{ id }} + vm_network: + cmd: 'destroy' + vm_set_id: "{{ id }}" + port1_bridge: "{{ port1_bridge }}" + vm_set_dict: "{{ VMs }}" + fp_mtu: "{{ fp_mtu_size }}" + ext_iface: "{{ external_iface }}" + vlan_base: "{{ vlan_base }}" diff --git a/ansible/start_ptf_containers.yml b/ansible/start_ptf_containers.yml index 8c29531a60a..39a3df63545 100644 --- a/ansible/start_ptf_containers.yml +++ b/ansible/start_ptf_containers.yml @@ -20,6 +20,10 @@ # # Then start a new container with new vlan range # ansible-playbook -i veos start_ptf_containers.yml --vault-password-file=~/.password --limit server_2 -e ptf_3=true -e ptf_3_vlan_base=381 +# +# To start a ptf container with saithrift packge deployed instead of raw ptf: +# ansible-playbook -i veos start_ptf_containers.yml --vault=password-file=~/.password --limit server_1 -e ptf_3=true -e docker_image=docker-ptf-sai-{ASIC vender name} + - hosts: servers:&vm_host gather_facts: no