Skip to content

Commit a0836c8

Browse files
authored
[testbed.csv] Add new fields (#2822)
What is the motivation for this PR? For server recovery script, we need the new fields inv_name to decide which lab inventory to use in recover tasks(deploy-mg), and we need the new fields auto_recover to decide whether we should recover this testbed.
1 parent e893f73 commit a0836c8

7 files changed

Lines changed: 79 additions & 56 deletions

File tree

ansible/library/test_facts.py

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,20 @@
2828
'''
2929

3030
EXAMPLES = '''
31-
Testbed CSV file example:
31+
Testbed CSV file example - deprecated:
3232
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,comment
3333
ptf1-m,ptf1,ptf32,docker-ptf,ptf-1,10.255.0.188/24,,server_1,,str-msn2700-01,Tests ptf
3434
vms-t1,vms1-1,t1,docker-ptf,ptf-2,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,Tests vms
3535
vms-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-3,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,Tests vms
3636
...
3737
38+
Testbed CSV file example - recommended:
39+
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,inv_file,auto_recover,comment
40+
ptf1-m,ptf1,ptf32,docker-ptf,ptf-1,10.255.0.188/24,,server_1,,str-msn2700-01,lab,False,Tests ptf
41+
vms-t1,vms1-1,t1,docker-ptf,ptf-2,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests vms
42+
vms-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-3,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests vms
43+
...
44+
3845
Testcases YAML File example:
3946
testcases:
4047
acl:
@@ -98,7 +105,8 @@
98105
class ParseTestbedTopoinfo():
99106
"""Parse the testbed file used to describe whole testbed info"""
100107

101-
TESTBED_FIELDS = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut', 'comment')
108+
TESTBED_FIELDS_DEPRECATED = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut', 'comment')
109+
TESTBED_FIELDS_RECOMMENDED = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut', 'inv_name', 'auto_recover', 'comment')
102110

103111
def __init__(self, testbed_file):
104112
self.testbed_filename = testbed_file
@@ -113,13 +121,17 @@ def _cidr_to_ip_mask(network):
113121
def _read_testbed_topo_from_csv():
114122
"""Read csv testbed info file."""
115123
with open(self.testbed_filename) as f:
116-
topo = csv.DictReader(f, fieldnames=self.TESTBED_FIELDS,
117-
delimiter=',')
118-
119-
# Validate all field are in the same order and are present
120-
header = next(topo)
121-
for field in self.TESTBED_FIELDS:
122-
assert header[field].replace('#', '').strip() == field
124+
header = [field.strip(' #') for field in f.readline().strip().split(',')]
125+
if len(header) == len(self.TESTBED_FIELDS_DEPRECATED):
126+
testbed_fields = self.TESTBED_FIELDS_DEPRECATED
127+
elif len(header) == len(self.TESTBED_FIELDS_RECOMMENDED):
128+
testbed_fields = self.TESTBED_FIELDS_RECOMMENDED
129+
else:
130+
raise ValueError('Unsupported testbed fields %s' % str(header))
131+
for header_field, expect_field in zip(header, testbed_fields):
132+
assert header_field == expect_field
133+
134+
topo = csv.DictReader(f, fieldnames=testbed_fields, delimiter=',')
123135

124136
for line in topo:
125137
if line['conf-name'].lstrip().startswith('#'):
@@ -133,7 +145,7 @@ def _read_testbed_topo_from_csv():
133145
_cidr_to_ip_mask(line["ptf_ipv6"])
134146

135147
line['duts'] = line['dut'].translate(string.maketrans("", ""), "[] ").split(';')
136-
line['duts_map'] = {dut:line['duts'].index(dut) for dut in line['duts']}
148+
line['duts_map'] = {dut: line['duts'].index(dut) for dut in line['duts']}
137149
del line['dut']
138150

139151
self.testbed_topo[line['conf-name']] = line

ansible/testbed.csv

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,comment
2-
ptf1-m,ptf1,ptf32,docker-ptf,ptf-unknown,10.255.0.188/24,,server_1,,str-msn2700-01,Test ptf Mellanox
3-
ptf2-b,ptf2,ptf64,docker-ptf,ptf-unknown,10.255.0.189/24,,server_1,,lab-s6100-01,Test ptf Broadcom
4-
vms-sn2700-t1,vms1-1,t1,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,Tests Mellanox SN2700 vms
5-
vms-sn2700-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,Tests Mellanox SN2700 vms
6-
vms-sn2700-t0,vms1-1,t0,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,Tests Mellanox SN2700 vms
7-
vms-s6000-t0,vms2-1,t0,docker-ptf,ptf-unknown,10.255.0.179/24,,server_1,VM0100,lab-s6000-01,Tests Dell S6000 vms
8-
vms-a7260-t0,vms3-1,t0-116,docker-ptf,ptf-unknown,10.255.0.180/24,,server_1,VM0100,lab-a7260-01,Tests Arista A7260 vms
9-
vms-s6100-t0,vms4-1,t0-64,docker-ptf,ptf-unknown,10.255.0.181/24,,server_1,VM0100,lab-s6100-01,Tests Dell S6100 vms
10-
vms-s6100-t1,vms4-1,t1-64,docker-ptf,ptf-unknown,10.255.0.182/24,,server_1,VM0100,lab-s6100-01,Tests Dell S6100 vms
11-
vms-s6100-t1-lag,vms5-1,t1-64-lag,docker-ptf,ptf-unknown,10.255.0.183/24,,server_1,VM0100,lab-s6100-01,Tests Dell S6100 vms
12-
vms-multi-dut,vms1-duts,ptf64,docker-ptf,ptf-unknown,10.255.0.184/24,,server_1,VM0100,[dut-host1;dut-host2],Example Multi DUTs testbed
13-
vms-example-ixia-1,vms6-1,t0-64,docker-ptf-ixia,example-ixia-ptf-1,10.0.0.30/32,,server_6,VM0600,example-s6100-dut-1,superman
1+
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,inv_name,auto_recover,comment
2+
ptf1-m,ptf1,ptf32,docker-ptf,ptf-unknown,10.255.0.188/24,,server_1,,str-msn2700-01,lab,False,Test ptf Mellanox
3+
ptf2-b,ptf2,ptf64,docker-ptf,ptf-unknown,10.255.0.189/24,,server_1,,lab-s6100-01,lab,False,Test ptf Broadcom
4+
vms-sn2700-t1,vms1-1,t1,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms
5+
vms-sn2700-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms
6+
vms-sn2700-t0,vms1-1,t0,docker-ptf,ptf-unknown,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests Mellanox SN2700 vms
7+
vms-s6000-t0,vms2-1,t0,docker-ptf,ptf-unknown,10.255.0.179/24,,server_1,VM0100,lab-s6000-01,lab,True,Tests Dell S6000 vms
8+
vms-a7260-t0,vms3-1,t0-116,docker-ptf,ptf-unknown,10.255.0.180/24,,server_1,VM0100,lab-a7260-01,lab,True,Tests Arista A7260 vms
9+
vms-s6100-t0,vms4-1,t0-64,docker-ptf,ptf-unknown,10.255.0.181/24,,server_1,VM0100,lab-s6100-01,lab,True,Tests Dell S6100 vms
10+
vms-s6100-t1,vms4-1,t1-64,docker-ptf,ptf-unknown,10.255.0.182/24,,server_1,VM0100,lab-s6100-01,lab,True,Tests Dell S6100 vms
11+
vms-s6100-t1-lag,vms5-1,t1-64-lag,docker-ptf,ptf-unknown,10.255.0.183/24,,server_1,VM0100,lab-s6100-01,lab,True,ests Dell S6100 vms
12+
vms-multi-dut,vms1-duts,ptf64,docker-ptf,ptf-unknown,10.255.0.184/24,,server_1,VM0100,[dut-host1;dut-host2],lab,True,Example Multi DUTs testbed
13+
vms-example-ixia-1,vms6-1,t0-64,docker-ptf-ixia,example-ixia-ptf-1,10.0.0.30/32,,server_6,VM0600,example-s6100-dut-1,lab,True,superman

ansible/vtestbed.csv

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,comment
2-
vms-kvm-t0,vms6-1,t0,docker-ptf,ptf-01,10.250.0.102/24,fec0::ffff:afa:2/64,server_1,VM0100,[vlab-01],Tests virtual switch vm
3-
vms-kvm-t0-64,vms6-1,t0-64,docker-ptf,ptf-01,10.250.0.102/24,fec0::ffff:afa:2/64,server_1,VM0100,[vlab-02],Tests virtual switch vm
4-
vms-kvm-t1-lag,vms6-2,t1-lag,docker-ptf,ptf-02,10.250.0.106/24,fec0::ffff:afa:6/64,server_1,VM0104,[vlab-03],Tests virtual switch vm
5-
vms-kvm-t0-2,vms6-3,t0,docker-ptf,ptf-03,10.250.0.108/24,fec0::ffff:afa:8/64,server_1,VM0104,[vlab-04],Tests virtual switch vm
6-
vms-kvm-dual-t0,vms6-4,dualtor,docker-ptf,ptf-04,10.250.0.109/24,fec0::ffff:afa:9/64,server_1,VM0108,[vlab-05;vlab-06],Dual-TOR testbed
7-
vms-kvm-multi-asic-t1-lag,vms6-4,t1-64-lag,docker-ptf,ptf-05,10.250.0.110/24,fec0::ffff:afa:a/64,server_1,VM0104,[vlab-07],Tests multi-asic virtual switch vm
1+
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,inv_name,auto_recover,comment
2+
vms-kvm-t0,vms6-1,t0,docker-ptf,ptf-01,10.250.0.102/24,fec0::ffff:afa:2/64,server_1,VM0100,[vlab-01],lab,False,Tests virtual switch vm
3+
vms-kvm-t0-64,vms6-1,t0-64,docker-ptf,ptf-01,10.250.0.102/24,fec0::ffff:afa:2/64,server_1,VM0100,[vlab-02],lab,False,Tests virtual switch vm
4+
vms-kvm-t1-lag,vms6-2,t1-lag,docker-ptf,ptf-02,10.250.0.106/24,fec0::ffff:afa:6/64,server_1,VM0104,[vlab-03],lab,False,Tests virtual switch vm
5+
vms-kvm-t0-2,vms6-3,t0,docker-ptf,ptf-03,10.250.0.108/24,fec0::ffff:afa:8/64,server_1,VM0104,[vlab-04],lab,False,Tests virtual switch vm
6+
vms-kvm-dual-t0,vms6-4,dualtor,docker-ptf,ptf-04,10.250.0.109/24,fec0::ffff:afa:9/64,server_1,VM0108,[vlab-05;vlab-06],lab,False,Dual-TOR testbed
7+
vms-kvm-multi-asic-t1-lag,vms6-4,t1-64-lag,docker-ptf,ptf-05,10.250.0.110/24,fec0::ffff:afa:a/64,server_1,VM0104,[vlab-07],lab,False,Tests multi-asic virtual switch vm

docs/testbed/README.new.testbed.Configuration.md

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,12 +49,12 @@ The devices section is a dictionary that contains all devices and hosts. This se
4949

5050
For each device that you add, add the following:
5151

52-
| Hostname | ansible_host | ansible_ssh_user | ansible_ssh_pass | HwSKU | device_type |
53-
| ------ | ------ | ------ | ------ | ------ | ------ |
54-
| str-msn2700-01 | [IP Address] | [username] | [password] | DevSonic | DevSonic |
55-
| str-7260-10 | [IP Address] | [username] | [password] |Arista-7260QX-64 | FanoutRoot |
56-
| str-7260-10 | [IP Address] | [username] | [password] |Arista-7260QX-64 | FanoutLeaf |
57-
| str-acs-serv-01 | [IP Address] | [username] | [password] | TestServ | Server |
52+
| Hostname | ansible_host | ansible_ssh_user | ansible_ssh_pass | HwSKU | device_type |
53+
| --------------- | ------------ | ---------------- | ---------------- | ---------------- | ----------- |
54+
| str-msn2700-01 | [IP Address] | [username] | [password] | DevSonic | DevSonic |
55+
| str-7260-10 | [IP Address] | [username] | [password] | Arista-7260QX-64 | FanoutRoot |
56+
| str-7260-10 | [IP Address] | [username] | [password] | Arista-7260QX-64 | FanoutLeaf |
57+
| str-acs-serv-01 | [IP Address] | [username] | [password] | TestServ | Server |
5858

5959
- hostname - names the devices you will use
6060
- ansible_host - this is the managementIP where you can connect to to the device
@@ -111,10 +111,10 @@ Define:
111111

112112
This is where the topology configuration file for the testbed will collect information from when running TestbedProcessing.py.
113113

114-
| #conf-name | group-name | topo | ptf_image_name | ptf_ip | server | vm_base | dut | comment |
115-
| ------ | ------ | ------ | ------ | ------ | ------ | ------ | ------ | ------ |
116-
| [ptf32 conf-name] | [ptf32 group-name] | [ptf32] | [docker-ptf] | [ip address] | [server group] | [vm_base] | [dut] | [comment] |
117-
| [t0 conf-name] | [t0 group-name] | [t0] | [docker-ptf] | [ip address] | [server group] | [vm_base] | [dut] | [comment] |
114+
| #conf-name | group-name | topo | ptf_image_name | ptf_ip | server | vm_base | dut | inv_name | auto_recover | comment |
115+
| ----------------- | ------------------ | ------- | -------------- | ------------ | -------------- | --------- | ----- | ---------- | -------------- | --------- |
116+
| [ptf32 conf-name] | [ptf32 group-name] | [ptf32] | [docker-ptf] | [ip address] | [server group] | [vm_base] | [dut] | [inv_name] | [auto_recover] | [comment] |
117+
| [t0 conf-name] | [t0 group-name] | [t0] | [docker-ptf] | [ip address] | [server group] | [vm_base] | [dut] | [inv_name] | [auto_recover] | [comment] |
118118

119119

120120
For each topology you use in your testbed environment, define the following:
@@ -129,6 +129,8 @@ For each topology you use in your testbed environment, define the following:
129129
- server - server where the testbed resides. Choose a veos_group to use that contains both the lab server and virtual machines
130130
- vm_base - enter in the lowest ID value for the VMs you will be using to run the test cases. The lowest VM ID value can be found under the veos section of the testbed configuration file. IF empty, no VMs are used
131131
- dut - enter in the target DUT that is used in the testbed environment
132+
- inv_name - inventory file name that contains the definition of the target DUTs
133+
- auto_recover - (`yes`|`True`|`true`) to recover this testbed when runnings serve recovery script, (`no`|`False`|`false`) otherwise
132134
- comment - make a little note here
133135
- ansible
134136
- ansible_host - IP address with port number

docs/testbed/README.testbed.Cli.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212

1313
## Add/Remove topo
1414
```
15-
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,comment
16-
vms1-1-t1,vms1-1,t1,docker-ptf,ptf-1,10.0.10.5/23,,server_1,VM0100,str-msn2700-11,t1 tests
17-
vms1-1-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-2,10.0.10.5/23,,server_1,VM0100,str-msn2700-11,t1-lag tests
15+
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,inv_name,auto_recover,comment
16+
vms1-1-t1,vms1-1,t1,docker-ptf,ptf-1,10.0.10.5/23,,server_1,VM0100,str-msn2700-11,lab,True,t1 tests
17+
vms1-1-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-2,10.0.10.5/23,,server_1,VM0100,str-msn2700-11,lab,False,t1-lag tests
1818
1919
```
2020
Goal is to use one VM with different topologies

docs/testbed/README.testbed.Config.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,10 @@
2525

2626
### ```testbed.csv``` format
2727
```
28-
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,comment
29-
ptf1-m,ptf1,ptf32,docker-ptf,ptf-1,10.255.0.188/24,,server_1,,str-msn2700-01,Tests ptf
30-
vms-t1,vms1-1,t1,docker-ptf,ptf-2,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,Tests vms
31-
vms-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-3,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,Tests vms
28+
# conf-name,group-name,topo,ptf_image_name,ptf,ptf_ip,ptf_ipv6,server,vm_base,dut,inv_name,auto_recover,comment
29+
ptf1-m,ptf1,ptf32,docker-ptf,ptf-1,10.255.0.188/24,,server_1,,str-msn2700-01,lab,False,Tests ptf
30+
vms-t1,vms1-1,t1,docker-ptf,ptf-2,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests vms
31+
vms-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-3,10.255.0.178/24,,server_1,VM0100,str-msn2700-01,lab,True,Tests vms
3232
3333
```
3434

@@ -40,6 +40,8 @@ vms-t1-lag,vms1-1,t1-lag,docker-ptf,ptf-3,10.255.0.178/24,,server_1,VM0100,str-m
4040
- server – server where the testbed resides
4141
- vm_base – first VM for the testbed. If empty, no VMs are used
4242
- dut – target dut name
43+
- inv_name - inventory file name that contains the definition of the target DUTs
44+
- auto_recover - (`yes`|`True`|`true`) to recover this testbed when runnings serve recovery script, (`no`|`False`|`false`) otherwise
4345
- comment – any text here
4446

4547
### ```testbed.csv``` consistency rules

tests/common/testbed.py

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
class TestbedInfo(object):
2222
"""Parse the testbed file used to describe whole testbed info."""
2323

24-
TESTBED_FIELDS = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut', 'comment')
24+
TESTBED_FIELDS_DEPRECATED = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut', 'comment')
25+
TESTBED_FIELDS_RECOMMENDED = ('conf-name', 'group-name', 'topo', 'ptf_image_name', 'ptf', 'ptf_ip', 'ptf_ipv6', 'server', 'vm_base', 'dut', 'inv_name', 'auto_recover', 'comment')
2526

2627
def __init__(self, testbed_file):
2728
if testbed_file.endswith(".csv"):
@@ -69,13 +70,18 @@ def _ip_mask_to_cidr(self, ip_address, netmask):
6970
def _read_testbed_topo_from_csv(self):
7071
"""Read csv testbed info file."""
7172
with open(self.testbed_filename) as f:
72-
topo = csv.DictReader(f, fieldnames=self.TESTBED_FIELDS,
73-
delimiter=',')
73+
header = [field.strip(' #') for field in f.readline().strip().split(',')]
74+
print(header)
75+
if len(header) == len(self.TESTBED_FIELDS_DEPRECATED):
76+
self.testbed_fields = self.TESTBED_FIELDS_DEPRECATED
77+
elif len(header) == len(self.TESTBED_FIELDS_RECOMMENDED):
78+
self.testbed_fields = self.TESTBED_FIELDS_RECOMMENDED
79+
else:
80+
raise ValueError('Unsupported testbed fields %s' % str(header))
81+
for header_field, expect_field in zip(header, self.testbed_fields):
82+
assert header_field == expect_field
7483

75-
# Validate all field are in the same order and are present
76-
header = next(topo)
77-
for field in self.TESTBED_FIELDS:
78-
assert header[field].replace('#', '').strip() == field
84+
topo = csv.DictReader(f, fieldnames=self.testbed_fields, delimiter=',')
7985

8086
for line in topo:
8187
if line['conf-name'].lstrip().startswith('#'):
@@ -89,7 +95,7 @@ def _read_testbed_topo_from_csv(self):
8995
self._cidr_to_ip_mask(line['ptf_ipv6'])
9096

9197
line['duts'] = line['dut'].translate(string.maketrans("", ""), "[] ").split(';')
92-
line['duts_map'] = {dut:line['duts'].index(dut) for dut in line['duts']}
98+
line['duts_map'] = {dut: line['duts'].index(dut) for dut in line['duts']}
9399
del line['dut']
94100

95101
self.testbed_topo[line['conf-name']] = line
@@ -135,6 +141,7 @@ class IncIndentDumper(yaml.Dumper):
135141
[1]: https://web.archive.org/web/20170903201521/https://pyyaml.org/ticket/64
136142
[2]: https://github.com/yaml/pyyaml/issues/127
137143
"""
144+
138145
def increase_indent(self, flow=False, indentless=False):
139146
return yaml.Dumper.increase_indent(self, flow, False)
140147

@@ -153,7 +160,7 @@ def write_line_break(self, data=None):
153160
ptf_ipv6 = self._ip_mask_to_cidr(tb_dict["ptf_ipv6"],
154161
tb_dict["ptf_netmask_v6"])
155162
testbed_mapping = zip(
156-
self.TESTBED_FIELDS,
163+
self.testbed_fields,
157164
[
158165
tb_name,
159166
tb_dict["group-name"],

0 commit comments

Comments
 (0)