Support yaml testbed file

ansible/library/test_facts.py

@@ -4,9 +4,8 @@
 import ipaddr as ipaddress
 import csv
 import string
-from operator import itemgetter
-from itertools import groupby
 import yaml
+
 from collections import defaultdict
 
 DOCUMENTATION = '''
@@ -97,52 +96,76 @@
 
 
 class ParseTestbedTopoinfo():
-    '''
-    Parse the CSV file used to describe whole testbed info
-    Please refer to the example of the CSV file format
-    CSV file first line is title
-    The topology name in title is using conf-name
-    '''
+    """Parse the testbed file used to describe whole testbed info"""
+
+    TESTBED_FIELDS = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut', 'comment')
+
     def __init__(self, testbed_file):
         self.testbed_filename = testbed_file
         self.testbed_topo = defaultdict()
 
     def read_testbed_topo(self):
-        CSV_FIELDS = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut', 'comment')
-        with open(self.testbed_filename) as f:
-            topo = csv.DictReader(f, fieldnames=CSV_FIELDS, delimiter=',')
-
-            # Validate all field are in the same order and are present
-            header = next(topo)
-            for field in CSV_FIELDS:
-                assert header[field].replace('#', '').strip() == field
-
-            for line in topo:
-                if line['conf-name'].lstrip().startswith('#'):
-                    ### skip comment line
-                    continue
-                if line['ptf_ip']:
-                    ptfaddress = ipaddress.IPNetwork(line['ptf_ip'])
-                    line['ptf_ip'] = str(ptfaddress.ip)
-                    line['ptf_netmask'] = str(ptfaddress.netmask)
-                if line['ptf_ipv6']:
-                    ptfaddress = ipaddress.IPNetwork(line['ptf_ipv6'])
-                    line['ptf_ipv6'] = str(ptfaddress.ip)
-                    line['ptf_netmask_v6'] = str(ptfaddress.netmask)
-
-                line['duts'] = line['dut'].translate(string.maketrans("", ""), "[] ").split(';')
-                line['duts_map'] = {dut:line['duts'].index(dut) for dut in line['duts']}
-                del line['dut']
-
-                self.testbed_topo[line['conf-name']] = line
-        return
+
+        def _cidr_to_ip_mask(network):
+            addr = ipaddress.IPNetwork(network)
+            return str(addr.ip), str(addr.netmask)
+
+        def _read_testbed_topo_from_csv():
+            """Read csv testbed info file."""
+            with open(self.testbed_filename) as f:
+                topo = csv.DictReader(f, fieldnames=self.TESTBED_FIELDS,
+                                      delimiter=',')
+
+                # Validate all field are in the same order and are present
+                header = next(topo)
+                for field in self.TESTBED_FIELDS:
+                    assert header[field].replace('#', '').strip() == field
+
+                for line in topo:
+                    if line['conf-name'].lstrip().startswith('#'):
+                        # skip comment line
+                        continue
+                    if line['ptf_ip']:
+                        line['ptf_ip'], line['ptf_netmask'] = \
+                            _cidr_to_ip_mask(line["ptf_ip"])
+                    if line['ptf_ipv6']:
+                        line['ptf_ipv6'], line['ptf_netmask_v6'] = \
+                            _cidr_to_ip_mask(line["ptf_ipv6"])
+
+                    line['duts'] = line['dut'].translate(string.maketrans("", ""), "[] ").split(';')
+                    line['duts_map'] = {dut:line['duts'].index(dut) for dut in line['duts']}
+                    del line['dut']
+
+                    self.testbed_topo[line['conf-name']] = line
+
+        def _read_testbed_topo_from_yaml():
+            """Read yaml testbed info file."""
+            with open(self.testbed_filename) as f:
+                tb_info = yaml.safe_load(f)
+                for tb in tb_info:
+                    if tb["ptf_ip"]:
+                        tb["ptf_ip"], tb["ptf_netmask"] = \
+                            _cidr_to_ip_mask(tb["ptf_ip"])
+                    if tb["ptf_ipv6"]:
+                        tb["ptf_ipv6"], tb["ptf_netmask_v6"] = \
+                            _cidr_to_ip_mask(tb["ptf_ipv6"])
+                    tb["duts"] = tb.pop("dut")
+                    tb["duts_map"] =  \
+                        {dut: i for i, dut in enumerate(tb["duts"])}
+                    self.testbed_topo[tb["conf-name"]] = tb
+
+        if self.testbed_filename.endswith(".csv"):
+            _read_testbed_topo_from_csv()
+        elif self.testbed_filename.endswith(".yaml"):
+            _read_testbed_topo_from_yaml()
 
     def get_testbed_info(self, testbed_name):
         if testbed_name:
-            return [self.testbed_topo[testbed_name]]
+            return self.testbed_topo[testbed_name]
         else:
             return self.testbed_topo
 
+
 class TestcasesTopology():
     '''
     Read testcases definition yaml file under ansible/roles/test/vars/testcases.yml
@@ -167,6 +190,7 @@ def read_testcases(self):
     def get_topo_testcase(self):
         return self.topo_testcase
 
+
 def main():
     module = AnsibleModule(
         argument_spec=dict(
@@ -183,7 +207,7 @@ def main():
     try:
         topoinfo = ParseTestbedTopoinfo(testbed_file)
         topoinfo.read_testbed_topo()
-        testbed_topo = topoinfo.get_testbed_info(testbed_name)[0]
+        testbed_topo = topoinfo.get_testbed_info(testbed_name)
         testcaseinfo = TestcasesTopology(testcase_file)
         testcaseinfo.read_testcases()
         testcase_topo = testcaseinfo.get_topo_testcase()
@@ -193,6 +217,7 @@ def main():
     except Exception as e:
         module.fail_json(msg=traceback.format_exc())
 
+
 from ansible.module_utils.basic import *
 if __name__== "__main__":
     main()

ansible/testbed-cli.sh

@@ -63,10 +63,8 @@ function usage
   exit
 }
 
-function read_file
+function read_csv
 {
-  echo reading
-
   # Filter testbed names in the first column in the testbed definition file
   line=$(cat $tbfile | grep "^$1,")
 
@@ -100,6 +98,59 @@ function read_file
   duts=${dut//[\[\] ]/}
 }
 
+function read_yaml
+{
+  content=$(python -c "from __future__ import print_function; import yaml; print('+'.join(str(tb) for tb in yaml.safe_load(open('$tbfile')) if '$1' in str(tb)))")
+
+  IFS=$'+' read -r -a tb_lines <<< $content
+  linecount=${#tb_lines[@]}
+
+  if [ $linecount == 0 ]
+  then
+    echo "Couldn't find topology name '$1'"
+    exit
+  elif [ $linecount -gt 1 ]
+  then
+    echo "Find more than one topology name in $tbfile"
+    exit
+  else
+    echo found topology $1
+  fi
+
+  tb_line=${tb_lines[0]}
+  line_arr=($1)
+  for attr in group-name topo ptf_image_name ptf ptf_ip ptf_ipv6 server vm_base dut comment;
+  do
+    value=$(python -c "from __future__ import print_function; tb=eval(\"$tb_line\"); print(tb['$attr'])")
+    [ "$value" == "None" ] && value=
+    line_arr=("${line_arr[@]}" "$value")
+  done
+
+  vm_set_name=${line_arr[1]}
+  topo=${line_arr[2]}
+  ptf_imagename=${line_arr[3]}
+  ptf=${line_arr[4]}
+  ptf_ip=${line_arr[5]}
+  ptf_ipv6=${line_arr[6]}
+  server=${line_arr[7]}
+  vm_base=${line_arr[8]}
+  dut=${line_arr[9]}
+  duts=$(python -c "from __future__ import print_function; print(','.join(eval(\"$dut\")))")
+}
+
+function read_file
+{
+  echo reading
+
+  if [[ $tbfile == *.csv ]]
+  then
+    read_csv ${topology}
+  elif [[ $tbfile == *.yaml ]]
+  then
+    read_yaml ${topology}
+  fi
+}
+
 function start_vms
 {
   server=$1

ansible/testbed.yaml

@@ -0,0 +1,158 @@
+---
+
+- conf-name: ptf1-m
+  group-name: ptf1
+  topo: ptf32
+  ptf_image_name: docker-ptf-sai-mlnx
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.188/24
+  ptf_ipv6:
+  server: server_1
+  vm_base:
+  dut:
+    - str-msn2700-01
+  comment: Test ptf Mellanox
+
+- conf-name: ptf2-b
+  group-name: ptf2
+  topo: ptf64
+  ptf_image_name: docker-ptf-sai-brcm
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.189/24
+  ptf_ipv6:
+  server: server_1
+  vm_base:
+  dut:
+    - lab-s6100-01
+  comment: Test ptf Broadcom
+
+- conf-name: vms-sn2700-t1
+  group-name: vms1-1
+  topo: t1
+  ptf_image_name: docker-ptf-sai-mlnx
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.178/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - str-msn2700-01
+  comment: Tests Mellanox SN2700 vms
+
+- conf-name: vms-sn2700-t1-lag
+  group-name: vms1-1
+  topo: t1-lag
+  ptf_image_name: docker-ptf-sai-mlnx
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.178/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - str-msn2700-01
+  comment: Tests Mellanox SN2700 vms
+
+- conf-name: vms-sn2700-t0
+  group-name: vms1-1
+  topo: t0
+  ptf_image_name: docker-ptf-sai-mlnx
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.178/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - str-msn2700-01
+  comment: Tests Mellanox SN2700 vms
+
+- conf-name: vms-s6000-t0
+  group-name: vms2-1
+  topo: t0
+  ptf_image_name: docker-ptf-sai-brcm
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.179/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - lab-s6000-01
+  comment: Tests Dell S6000 vms
+
+- conf-name: vms-a7260-t0
+  group-name: vms3-1
+  topo: t0-116
+  ptf_image_name: docker-ptf-sai-brcm
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.180/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - lab-a7260-01
+  comment: Tests Arista A7260 vms
+
+- conf-name: vms-s6100-t0
+  group-name: vms4-1
+  topo: t0-64
+  ptf_image_name: docker-ptf-sai-brcm
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.181/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - lab-s6100-01
+  comment: Tests Dell S6100 vms
+
+- conf-name: vms-s6100-t1
+  group-name: vms4-1
+  topo: t1-64
+  ptf_image_name: docker-ptf-sai-brcm
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.182/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - lab-s6100-01
+  comment: Tests Dell S6100 vms
+
+- conf-name: vms-s6100-t1-lag
+  group-name: vms5-1
+  topo: t1-64-lag
+  ptf_image_name: docker-ptf-sai-brcm
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.183/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - lab-s6100-01
+  comment: Tests Dell S6100 vms
+
+- conf-name: vms-multi-dut
+  group-name: vms1-duts
+  topo: ptf64
+  ptf_image_name: docker-ptf
+  ptf: ptf-unknown
+  ptf_ip: 10.255.0.184/24
+  ptf_ipv6:
+  server: server_1
+  vm_base: VM0100
+  dut:
+    - dut-host1
+    - dut-host2
+  comment: Example Multi DUTs testbed
+
+- conf-name: vms-example-ixia-1
+  group-name: vms6-1
+  topo: t0-64
+  ptf_image_name: docker-ptf-ixia
+  ptf: example-ixia-ptf-1
+  ptf_ip: 10.0.0.30/32
+  ptf_ipv6:
+  server: server_6
+  vm_base: VM0600
+  dut:
+    - example-s6100-dut-1
+  comment: superman