diff --git a/ansible/roles/test/files/ptftests/advanced-reboot.py b/ansible/roles/test/files/ptftests/advanced-reboot.py index 2fc6a7edcb4..2bc7cb777e3 100644 --- a/ansible/roles/test/files/ptftests/advanced-reboot.py +++ b/ansible/roles/test/files/ptftests/advanced-reboot.py @@ -311,7 +311,7 @@ def get_portchannel_info(self): for vm_key in self.vm_dut_map.keys(): if member in self.vm_dut_map[vm_key]['dut_ports']: self.vm_dut_map[vm_key]['dut_portchannel'] = key - self.vm_dut_map[vm_key]['neigh_portchannel'] = 'Port-Channel 1' + self.vm_dut_map[vm_key]['neigh_portchannel'] = 'Port-Channel1' break def get_neigh_port_info(self): @@ -732,7 +732,7 @@ def wait_for_ssh_threads(): if self.reboot_type == 'warm-reboot' and self.preboot_oper is not None: if self.pre_handle is not None: self.log("Postboot checks:") - log_info, fails_dut, fails_vm = self.pre_handle.verify() + log_info, fails_dut, fails_vm = self.pre_handle.verify(pre_check=False) self.fails[self.neigh_vm] |= fails_vm self.fails['dut'] |= fails_dut for log in log_info: diff --git a/ansible/roles/test/files/ptftests/arista.py b/ansible/roles/test/files/ptftests/arista.py index 3ddbf1bb178..a8aaa0fa738 100644 --- a/ansible/roles/test/files/ptftests/arista.py +++ b/ansible/roles/test/files/ptftests/arista.py @@ -358,7 +358,7 @@ def verify_bgp_neigh_state(self, dut=None, state="Active"): data = '\n'.join(output.split('\r\n')[1:-1]) obj = json.loads(data) - if state != 'Active': + if state == 'down': if 'vrfs' in obj: # return True when obj['vrfs'] is empty which is the case when the bgp state is 'down' bgp_state[ver] = not obj['vrfs'] @@ -375,6 +375,36 @@ def verify_bgp_neigh_state(self, dut=None, state="Active"): self.fails.add('Verify BGP %s neighbor: Object missing in output' % ver) return self.fails, bgp_state + def change_neigh_lag_state(self, lag, is_up=True): + state = ['shut', 'no shut'] + self.do_cmd('configure') + is_match = re.match('(Port-Channel|Ethernet)\d+', lag) + if is_match: + output = self.do_cmd('interface %s' % lag) + if 'Invalid' not in output: + self.do_cmd(state[is_up]) + self.do_cmd('exit') + self.do_cmd('exit') + + def verify_neigh_lag_state(self, lag, state="connected", pre_check=True): + lag_state = False + msg_prefix = ['Postboot', 'Preboot'] + is_match = re.match('(Port-Channel|Ethernet)\d+', lag) + if is_match: + output = self.do_cmd('show interfaces %s | json' % lag) + if 'Invalid' not in output: + data = '\n'.join(output.split('\r\n')[1:-1]) + obj = json.loads(data) + + if 'interfaces' in obj and lag in obj['interfaces']: + lag_state = (obj['interfaces'][lag]['interfaceStatus'] == state) + else: + self.fails.add('%s: Verify LAG %s: Object missing in output' % (msg_prefix[pre_check], lag)) + return self.fails, lag_state + + self.fails.add('%s: Invalid interface name' % msg_prefix[pre_check]) + return self.fails, lag_state + def check_gr_peer_status(self, output): # [0] True 'ipv4_gr_enabled', [1] doesn't matter 'ipv6_enabled', [2] should be >= 120 if not self.ipv4_gr_enabled: diff --git a/ansible/roles/test/files/ptftests/sad_path.py b/ansible/roles/test/files/ptftests/sad_path.py index ccae8ad0878..fec1f64fd41 100644 --- a/ansible/roles/test/files/ptftests/sad_path.py +++ b/ansible/roles/test/files/ptftests/sad_path.py @@ -20,12 +20,13 @@ def __init__(self, oper_type, vm_list, portchannel_ports, vm_dut_map, test_args, self.shandle = SadOper(self.oper_type, self.vm_list, self.portchannel_ports, self.vm_dut_map, self.test_args, self.dut_ssh) def setup(self): - if 'bgp' in self.oper_type: - self.shandle.sad_setup(is_up=False) + self.shandle.sad_setup(is_up=False) return self.shandle.retreive_test_info(), self.shandle.retreive_logs() - def verify(self): + def verify(self, pre_check=True): self.shandle.sad_bgp_verify() + if 'lag' in self.oper_type: + self.shandle.sad_lag_verify(pre_check=pre_check) return self.shandle.retreive_logs() def revert(self): @@ -111,6 +112,9 @@ def __init__(self, oper_type, vm_list, portchannel_ports, vm_dut_map, test_args, super(SadOper, self).__init__(oper_type, vm_list, portchannel_ports, vm_dut_map, test_args) self.dut_ssh = dut_ssh self.dut_needed = None + self.lag_members_down = None + self.neigh_lag_state = None + self.msg_prefix = ['Postboot', 'Preboot'] def populate_bgp_state(self): if self.oper_type == 'neigh_bgp_down': @@ -121,19 +125,42 @@ def populate_bgp_state(self): self.neigh_bgp['changed_state'] = 'Active' self.dut_bgp['changed_state'] = 'Idle' self.dut_needed = self.dut_bgp + elif self.oper_type == 'neigh_lag_down': + # on the DUT side, bgp states are different pre and post boot. hence passing multiple values + self.neigh_bgp['changed_state'] = 'Idle' + self.dut_bgp['changed_state'] = 'Connect,Active,Idle' + self.dut_needed = self.dut_bgp + elif self.oper_type == 'dut_lag_down': + self.neigh_bgp['changed_state'] = 'Idle' + self.dut_bgp['changed_state'] = 'Active,Connect,Idle' + self.dut_needed = self.dut_bgp def sad_setup(self, is_up=True): self.log = [] + if not is_up: self.setup() self.populate_bgp_state() - self.log.append('BGP state change will be for %s' % self.neigh_vm) - if self.oper_type == 'neigh_bgp_down': - self.log.append('Changing state of AS %s to shut' % self.neigh_bgp['asn']) - self.vm_handle.change_bgp_neigh_state(self.neigh_bgp['asn'], is_up=is_up) - elif self.oper_type == 'dut_bgp_down': - self.change_bgp_dut_state(is_up=is_up) - time.sleep(30) + if 'lag' in self.oper_type: + self.populate_lag_state() + + if 'bgp' in self.oper_type: + self.log.append('BGP state change will be for %s' % self.neigh_vm) + if self.oper_type == 'neigh_bgp_down': + self.log.append('Changing state of AS %s to shut' % self.neigh_bgp['asn']) + self.vm_handle.change_bgp_neigh_state(self.neigh_bgp['asn'], is_up=is_up) + elif self.oper_type == 'dut_bgp_down': + self.change_bgp_dut_state(is_up=is_up) + time.sleep(30) + elif 'lag' in self.oper_type: + self.log.append('LAG state change will be for %s' % self.neigh_vm) + if self.oper_type == 'neigh_lag_down': + self.log.append('Changing state of LAG %s to shut' % self.vm_dut_map[self.neigh_name]['neigh_portchannel']) + self.vm_handle.change_neigh_lag_state(self.vm_dut_map[self.neigh_name]['neigh_portchannel'], is_up=is_up) + elif self.oper_type == 'dut_lag_down': + self.change_dut_lag_state(is_up=is_up) + # wait for sometime for lag members state to sync + time.sleep(120) def change_bgp_dut_state(self, is_up=True): state = ['shutdown', 'startup'] @@ -149,18 +176,19 @@ def change_bgp_dut_state(self, is_up=True): self.fails['dut'].add('Stderr: %s' % stderr) def verify_bgp_dut_state(self, state='Idle'): + states = state.split(',') bgp_state = {} bgp_state['v4'] = bgp_state['v6'] = False for key in self.neigh_bgp.keys(): if key not in ['v4', 'v6']: continue - self.log.append('Verifying if the DUT side BGP peer %s is %s' % (self.neigh_bgp[key], state)) + self.log.append('Verifying if the DUT side BGP peer %s is %s' % (self.neigh_bgp[key], states)) stdout, stderr, return_code = self.cmd(['ssh', '-oStrictHostKeyChecking=no', self.dut_ssh, 'show ip bgp neighbor %s' % self.neigh_bgp[key]]) if return_code == 0: for line in stdout.split('\n'): if 'BGP state' in line: curr_state = re.findall('BGP state = (\w+)', line)[0] - bgp_state[key] = (curr_state == state) + bgp_state[key] = (curr_state in states) break else: self.fails['dut'].add('Retreiving BGP info for peer %s from DUT side failed' % self.neigh_bgp[key]) @@ -181,3 +209,68 @@ def sad_bgp_verify(self): self.log.append('BGP state down as expected on DUT') else: self.fails['dut'].add('BGP state not down on DUT') + + def populate_lag_state(self): + if self.oper_type == 'neigh_lag_down': + self.neigh_lag_state = 'disabled' + self.lag_members_down = self.vm_dut_map[self.neigh_name]['dut_ports'] + elif self.oper_type == 'dut_lag_down': + self.lag_members_down = self.vm_dut_map[self.neigh_name]['dut_ports'] + self.neigh_lag_state = 'notconnect' + + def change_dut_lag_state(self, is_up=True): + state = ['shutdown', 'startup'] + dut_portchannel = self.vm_dut_map[self.neigh_name]['dut_portchannel'] + if not re.match('(PortChannel|Ethernet)\d+', dut_portchannel): return + self.log.append('Changing state of %s from DUT side to %s' % (dut_portchannel, state[is_up])) + stdout, stderr, return_code = self.cmd(['ssh', '-oStrictHostKeyChecking=no', self.dut_ssh, 'sudo config interface %s %s' % (state[is_up], dut_portchannel)]) + if return_code != 0: + self.fails['dut'].add('%s: State change not successful from DUT side for %s' % (self.msg_prefix[1 - is_up], dut_portchannel)) + self.fails['dut'].add('%s: Return code: %d' % (self.msg_prefix[1 - is_up], return_code)) + self.fails['dut'].add('%s: Stderr: %s' % (self.msg_prefix[1 - is_up], stderr)) + else: + self.log.append('State change successful on DUT') + + def verify_dut_lag_member_state(self, lag_memb_output, pre_check=True): + success = True + for member in self.vm_dut_map[self.neigh_name]['dut_ports']: + if self.lag_members_down is not None and member in self.lag_members_down: + search_str = '%s(D)' % member + else: + search_str = '%s(S)' % member + + if lag_memb_output.find(search_str) != -1: + self.log.append('Lag member %s state as expected' % member) + else: + success = False + self.fails['dut'].add('%s: Lag member %s state not as expected' % (self.msg_prefix[pre_check], member)) + return success + + def verify_dut_lag_state(self, pre_check=True): + pat = re.compile(".*%s\s+LACP\(A\)\(Dw\)\s+(.*)" % self.vm_dut_map[self.neigh_name]['dut_portchannel']) + stdout, stderr, return_code = self.cmd(['ssh', '-oStrictHostKeyChecking=no', self.dut_ssh, 'show interfaces portchannel']) + if return_code == 0: + for line in stdout.split('\n'): + if self.vm_dut_map[self.neigh_name]['dut_portchannel'] in line: + is_match = pat.match(line) + if is_match and self.verify_dut_lag_member_state(is_match.group(1), pre_check=pre_check): + self.log.append('Lag state is down as expected on the DUT') + self.log.append('Pattern check: %s' % line) + else: + self.fails['dut'].add('%s: Lag state is not down on the DUT' % self.msg_prefix[pre_check]) + self.fails['dut'].add('%s: Obtained: %s' % (self.msg_prefix[pre_check], line)) + break + else: + self.fails['dut'].add('%s: Retreiving LAG info from DUT side failed' % self.msg_prefix[pre_check]) + self.fails['dut'].add('%s: Return code: %d' % (self.msg_prefix[pre_check], return_code)) + self.fails['dut'].add('%s: Stderr: %s' % (self.msg_prefix[pre_check], stderr)) + + def sad_lag_verify(self, pre_check=True): + fails_vm, lag_state = self.vm_handle.verify_neigh_lag_state(self.vm_dut_map[self.neigh_name]['neigh_portchannel'], state=self.neigh_lag_state, pre_check=pre_check) + self.fails[self.neigh_vm] |= fails_vm + if lag_state: + self.log.append('LAG state down as expected for %s' % self.neigh_vm) + else: + self.fails[self.neigh_vm].add('%s: LAG state not down for %s' % (self.msg_prefix[pre_check], self.neigh_vm)) + self.log.append('Verifying LAG state on the dut end') + self.verify_dut_lag_state(pre_check=pre_check) diff --git a/ansible/roles/test/tasks/warm-reboot-sad.yml b/ansible/roles/test/tasks/warm-reboot-sad.yml index 920cae835a7..6ace17b74e0 100644 --- a/ansible/roles/test/tasks/warm-reboot-sad.yml +++ b/ansible/roles/test/tasks/warm-reboot-sad.yml @@ -7,5 +7,5 @@ include: advanced-reboot.yml vars: reboot_type: warm-reboot - preboot_list: ['neigh_bgp_down', 'dut_bgp_down'] + preboot_list: ['neigh_bgp_down', 'dut_bgp_down', 'dut_lag_down', 'neigh_lag_down'] preboot_files: "peer_dev_info,neigh_port_info"