[TestLAG] add additional/alternative lag_2 test (#199)

* add additional/alternative lag test for using lab graph file to identify fanout switch ports; modify the lacp_time playbook be able to validate both fast and slow rate; leave traffic balancing part of lag test to FIB test

* modify it run for T0 lag too

* add todo and fix typo

* Pretty much sure lacp timing issue is covered by lag_2.yml, all other existing lab tests, not very sure if they are still working or not since cannot test in my environment. Will leave it here until I can confirm no one is running it

* minor typo fix

* fix comment

Author: maggiemsft

ansible/group_vars/lab/labinfo.json

@@ -3,6 +3,7 @@
         "Arista-7050-Q4S48": "Arista",
         "Arista-7508-Q288": "Arista",
         "Arista-7260QX-64": "Arista",
+        "Arista-VM": "Arista",
         "Nexus-3064-NX": "Nexus",
         "Force10-S6100": "Force10",
         "Force10-S6000": "Force10"
@@ -16,7 +17,7 @@
         "Arista": {
             "user": "admin",
             "passwd": ["password", "123456"],
-            "enable": null
+            "enable": ['', null]
         },
         "Force10": {
             "user": "admin",

ansible/library/lag_facts.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python
+
+import json
+from ansible.module_utils.basic import *
+
+DOCUMENTATION = '''
+---
+module: lag_facts
+Ansible_version_added: "2.0.0.2"
+Sonic_version: "2.0"
+short_description: Retrieve lag(LACP) information from a device 
+description:
+    - Retrieved facts will be inserted to:
+        lag_facts:
+          - 'names': [list all portchannel names] 
+          - 'lags':  {portchannel: detailed portchannel information }
+'''
+
+EXAMPLES = '''
+# Gather lab facts
+ - name: Gather lag info
+   lag_facts:  host=10.255.0.200 
+ - name: display lag information
+   debug: var=lag_facts
+'''
+
+class LagModule(object):
+    def __init__(self):
+        self.module = AnsibleModule(
+            argument_spec=dict(
+                host=dict(required=True),
+            ),
+            supports_check_mode=False,
+        )
+        self.lag_names = []
+        self.lags = {}
+        return
+
+    def run(self):
+        '''
+            Main method of the class
+        '''
+        self.get_po_names()
+        for po in self.lag_names:
+            self.lags[po] = {}
+            self.lags[po]['po_stats'] = self.get_po_status(po)
+            self.lags[po]['po_config'] = self.get_po_config(po)
+            self.lags[po]['po_intf_stat'] = self.get_po_intf_stat(po)
+        self.module.exit_json(ansible_facts={'lag_facts': {'names': self.lag_names, 'lags': self.lags}})
+        return
+
+    def get_po_names(self):
+        '''
+            Collect configured lag interface names
+        '''
+        rt, out, err = self.module.run_command("sonic-cfggen -m /etc/sonic/minigraph.xml -v \"minigraph_portchannels.keys() | join(' ')\"")
+        if rt != 0:
+            fail_msg="Command to retrieve portchannel names failed return=%d, out=%s, err=%s" %(rt, out, err)
+            self.module.fail_json(msg=fail_msg)
+        else:
+            self.lag_names = out.split()
+        return
+
+    def get_po_status(self, po_name):
+        '''
+            Collect lag information by command docker teamdctl
+        '''
+        rt, out, err = self.module.run_command("docker exec -i teamd teamdctl "+po_name+" state dump")
+        if rt != 0:
+            fail_msg="failed dump port channel %s status return=%d, out=%s, err=%s" %(po_name, rt, out, err)
+            self.module.fail_json(msg=fail_msg)
+        json_info = json.loads(out)
+        return json_info
+
+    def get_po_config(self, po_name):
+        '''
+            Collect lag information by command docker teamdctl
+        '''
+        rt, out, err = self.module.run_command("docker exec -i teamd teamdctl "+po_name+" config dump")
+        if rt != 0:
+            fail_msg="failed dump port channel %s config return=%d, out=%s, err=%s" %(po_name, rt, out, err)
+            self.module.fail_json(msg=fail_msg)
+        json_info = json.loads(out)
+        return json_info
+
+    def get_po_intf_stat(self, po_name):
+        '''
+            Collect lag information by command docker teamdctl
+        '''
+        rt, out, err = self.module.run_command("ip link show " + po_name)
+        if rt != 0:
+            fail_msg="failed show interface status of %s return=%d, out=%s, err=%s" %(po_name, rt, out, err)
+            self.module.fail_json(msg=fail_msg)
+        if 'NO-CARRIER' in out:
+            return 'Down'
+        else:
+            return 'Up'
+
+def main():
+    lags = LagModule()
+    lags.run()
+
+if __name__ == '__main__':
+    main()

ansible/roles/test/tasks/lag_2.yml

@@ -0,0 +1,65 @@
+### this is the Lag_2 lag test that tests each Lag interface minimum link and rate of sending LACP DU packets
+### this test could be consider as an additional/alternative lag test from existing lagall.yml.
+### Due to some labs are using two layer fanout switches, and one DUT might connects to multiple fanoutleaf switches
+### so for minimum link test of lag member flaps, it requires to use lab connection facts to determine the fanout neighbor ports, 
+### Also, most of the traffic load balancing tests of LAG interface are covered in new FIB tests. so we are ignoring traffic test 
+### for lag member flaps for now, will consider add traffic back if required
+
+- fail: msg="Please define ptf_host"
+  when: ptf_host is not defined
+
+- fail: msg="Please define testbed_type"
+  when: testbed_type is not defined
+
+
+- fail: msg="this test only support t1-lag and t0 testbed_type"
+  when: testbed_type not in ['t1-lag', 't0']
+
+- name: gathering lag facts from device
+  lag_facts: host={{ inventory_hostname }}
+
+- fail: msg="No lag configuration found in {{ inventory_hostname }}"
+  when: lag_facts.names == [] 
+
+- name: Gathering peer VM information from lldp
+  lldp:
+  vars:
+    ansible_shell_type: docker
+    ansible_python_interpreter: docker exec -i lldp python
+
+- name: gathering minigraph of the device configuration
+  minigraph_facts: host={{ inventory_hostname }}
+  connection: local
+
+- name: Gathering lab graph facts about the device
+  conn_graph_facts: host={{ inventory_hostname }}
+  connection: local
+
+- set_fact:
+    fanout_neighbors: "{{device_conn}}"   
+
+- set_fact:
+    vm_neighbors: "{{ minigraph_neighbors }}"   
+
+- name: Copy PTF test into PTF-docker for test LACP DU.
+  copy: src=roles/test/files/acstests/{{ item }} dest=/tmp/{{ item }}
+  with_items:
+    - lag_test.py
+    - acs_base_test.py
+    - router_utils.py
+  delegate_to: "{{ ptf_host }}"
+
+- name: Include testbed topology configuration (to get LAG IP and PTF docker interfaces, that are behind LAG VMs).
+  include_vars: vars/topo_t1-lag.yml
+  when: testbed_type == 't1-lag'
+
+- name: Include testbed topology configuration (to get LAG IP and PTF docker interfaces, that are behind LAG VMs).
+  include_vars: vars/topo_t0.yml
+  when: testbed_type == 't0'
+
+- set_fact:
+    dut_mac: "{{ ansible_Ethernet0['macaddress'] }}"
+
+- name: test each lag interface minimum links and rate 
+  include: single_lag_test.yml
+  with_items: lag_facts.names

ansible/roles/test/tasks/lag_lacp_timing_test.yml

@@ -1,17 +1,19 @@
-#-----------------------------------
-# --- Part of lagall.yml test ---
-#-----------------------------------
+#--------------------------------------------------
+# --- lag_2.yml test---
+#-------------------------------------------------
 # Using PTF test framework, check the LACP packets timing
 # for each VM that has LAG configured.
 #
 # @ params: iface_behind_lag_member_0   -   PTF docker iface, that receives all the packets that VM LAG member does.
 # @ params: iface_behind_lag_member_1   -   PTF docker iface, that receives all the packets that VM LAG member does.
 # @ params: vm_name     -       VM hostname that will be printed before running test.
-#-----------------------------------
+# 
+# Originally in lagall.yml, and lag_2.yml should cover it 
+#--------------------------------------------------------
 
 - set_fact:
     lacp_ether_type: '0x8809'
-    packet_timing: 30
+    packet_timing: "{{ lacp_timer }}"
     packet_timeout: 35
 
 - name: Check LACP timing on eth{{ iface_behind_lag_member_0 }} (interface behind {{ vm_name }}).
@@ -27,3 +29,4 @@
     lag_ptf_test_name: LacpTimingTest
     params: "exp_iface={{ iface_behind_lag_member_1 }}; timeout={{ packet_timeout }}; packet_timing={{ packet_timing }}; ether_type={{ lacp_ether_type }}"
     change_dir: /tmp
+  when: iface_behind_lab_member_1 is defined

ansible/roles/test/tasks/lag_minlink.yml

@@ -0,0 +1,57 @@
+### This playbook is part of lag_2.yml test
+### It is to test LACP LAG functionality behave correctly when link flaps. 
+### In this playbook the neighbor of the selected flap_intf interface is shutdown and playbook validate Portchannel interface status and member
+### selection status are correct based on port channel min link configuration. Then bring up the remote interface to make sure
+### Port channel interface is up after peer port is back
+
+- block:
+    - name: Shut down neighbor interface {{ neighbor_interface }} on {{ peer_device }}
+      action: apswitch template=neighbor_interface_shut_single.j2
+      args:
+        host: "{{peer_host}}"
+        login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
+      connection: switch
+    
+    - pause:
+        seconds: "{{ wait_down_time }}"
+    
+    - lag_facts: host={{ inventory_hostname }}
+    
+    - name: Verify all other lag member interfaces are marked selected
+      assert: { that: "'{{ lag_facts.lags[po]['po_stats']['ports'][item]['runner']['selected'] }}' == 'True'" }
+      with_items: "{{ po_interfaces.keys() }}"
+      when: item != "{{ flap_intf }}"
+
+    - name: Verify {{ flap_intf}}  lag member interfaces are marked as deselected for the shutdown port
+      assert: { that: "'{{ lag_facts.lags[po]['po_stats']['ports'][item]['runner']['selected'] }}' == 'False'" }
+      with_items: "{{ po_interfaces.keys() }}"
+      when: item == "{{ flap_intf }}"
+
+    - name: verify port-channel {{ po }} interface are marked down correctly if portchannel should down
+      assert: { that: "'{{ lag_facts.lags[po]['po_intf_stat'] }}' == 'Down' "}
+      when: po_flap == True
+
+    - name: verify port-channel {{ po }} interface are marked Up correctly if portchannel should keepup
+      assert: { that: "'{{ lag_facts.lags[po]['po_intf_stat'] }}' == 'Up' "}
+      when: po_flap != True
+
+  ### always bring back port in case test error and left testbed in unknow stage
+  always:
+    - name: Bring up neighbor interface {{ neighbor_interface }} on {{ peer_host }}
+      action: apswitch template=neighbor_interface_no_shut_single.j2
+      args:
+        host: "{{peer_host}}"
+        login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
+      connection: switch
+    
+    - pause:
+        seconds: 20
+    
+    - lag_facts: host={{ inventory_hostname }}
+    
+    - name: Verify all interfaces in port_channel  are marked up
+      assert: { that: "'{{ lag_facts.lags[po]['po_stats']['ports'][item]['link']['up'] }}' == 'True'" }
+      with_items: "{{ po_interfaces.keys() }}"
+
+    - name: verify port-channel {{ po }} interface are marked up correctly
+      assert: { that: "'{{ lag_facts.lags[po]['po_intf_stat'] }}' == 'Up' "}

ansible/roles/test/tasks/lagall.yml

@@ -135,11 +135,3 @@
     dut_lag_member: "{{ dut_lag_members[1] }}"
     num_of_pkts: 100
   when: dut_lag_members is defined
-
-- name: --TEST-- LACP timing test.
-  include: lag_lacp_timing_test.yml
-  vars:
-    iface_behind_lag_member_0: "{{ topology['VMs'][item]['vlans'][0] }}"
-    iface_behind_lag_member_1: "{{ topology['VMs'][item]['vlans'][1] }}"
-    vm_name: "{{ item }}"
-  with_items: lag_vms.keys()

ansible/roles/test/tasks/single_lag_test.yml

@@ -0,0 +1,101 @@
+### Part of lag test palybook lag_2.yml (--tag lag_2)
+### This playbook test one single port channel minimum link feature of one member interface shutdown 
+### and portchannel member interfaces sending ACP DU rate
+
+# Gather information of port channel ports, minimum links and total interface member numbers
+- set_fact: 
+    po: "{{ item }}"
+    po_interfaces: "{{ lag_facts.lags[item]['po_config']['ports'] }}"
+    po_intf_num: "{{ lag_facts.lags[item]['po_config']['ports']|length }}"
+    po_min_links: "{{lag_facts.lags[item]['po_config']['runner']['min_ports']}}"
+
+# pick flap interface name and calculate when it flaps, should portchannel interface flap or not
+# Current it is using a static capacity < 75%, Portchannel will flap which match Sonic configuration
+# if need to be random, then will make it a var
+- set_fact:
+    po_flap: "{{ (po_intf_num|float - 1)/(po_min_links|float)*100 < 75 }}"
+    flap_intf: "{{ lag_facts.lags[item]['po_config']['ports'].keys()[0] }}"
+
+### figure out fanout switches info for the flapping lag member and run minlink test
+- set_fact:
+    peer_device: "{{ fanout_neighbors[flap_intf]['peerdevice'] }}"
+    neighbor_interface: "{{ fanout_neighbors[flap_intf]['peerport'] }}"
+
+- conn_graph_facts: host={{ peer_device }}
+  connection: local
+
+- set_fact:
+    peer_host: "{{ device_info['mgmtip'] }}"
+    peer_hwsku: "{{ device_info['HwSku'] }}"
+
+- name: test fanout interface (physical) flap and lacp keep correct po status follow minimum links requirement
+  include: lag_minlink.yml
+  vars:
+    wait_down_time: 20
+
+### Now figure out remote VM and interface info for the falpping lag member and run minlink test 
+- set_fact:
+    peer_device: "{{vm_neighbors[flap_intf]['name']}}"
+    neighbor_interface: "{{vm_neighbors[flap_intf]['port']}}"
+    peer_hwsku: 'Arista-VM'
+
+##############################################################################################
+##### TODO:  use minigraph/and VM configuration to figure out neighbor host access info #####
+#####        try not using lldp dynamic info to find neighbor access
+##############################################################################################
+- set_fact:
+    peer_host: "{{ lldp[flap_intf]['chassis']['mgmt-ip'] }}"
+
+- name: test vm interface flap (no physical port down, more like remote port lock) that lag interface can change to correct po status follow minimum links requirement
+  include: lag_minlink.yml
+  vars: 
+    wait_down_time: 120
+
+### Now prepare for the remote VM interfaces that using PTF docker to check teh LACP DU packet rate is correct 
+- set_fact:
+    iface_behind_lag_member_0: "{{ topology['VMs'][peer_device]['vlans'][0] }}"
+
+- set_fact:
+    iface_behind_lag_member_1: "{{ topology['VMs'][peer_device]['vlans'][1] }}"
+  when: testbed_type == 't1-lag'
+
+- set_fact:
+    neighbor_lag_intfs: []
+
+- set_fact:
+    neighbor_lag_intfs: "{{ neighbor_lag_intfs }} + [ '{{ vm_neighbors[item]['port'] }}' ]"
+  with_items: "{{ po_interfaces }}"
+ 
+# make sure portchannel peer rate is set to fast
+- name: make sure all lag members on VM are set to fast
+  action: apswitch template=neighbor_lag_rate_fast.j2
+  args:
+    host: "{{peer_host}}"
+    login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
+  connection: switch
+
+- pause: 
+    seconds: 5
+ 
+- name: test lacp packet sending rate is 1 seconds
+  include: lag_lacp_timing_test.yml
+  vars: 
+    vm_name: "{{ peer_device }}"
+    lacp_timer: 1
+
+# make sure portchannel peer rate is set to slow
+- name: make sure all lag members on VM are set to slow
+  action: apswitch template=neighbor_lag_rate_slow.j2
+  args:
+    host: "{{peer_host}}"
+    login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
+  connection: switch
+
+- pause: 
+    seconds: 5
+ 
+- name: test lacp packet sending rate is 30 seconds
+  include: lag_lacp_timing_test.yml
+  vars: 
+    vm_name: "{{ peer_device }}"
+    lacp_timer: 30