[test] Add check neighbors health for SONiC neighbor devices

docs/testbed/README.testbed.VsSetup.md

@@ -293,8 +293,16 @@ cd sonic-mgmt/tests
 
 2. Run the following command to execute the `bgp_fact` test (including the pre/post setup steps):
 
+If neighbor devices are EOS
+
 ```
 ./run_tests.sh -n vms-kvm-t0 -d vlab-01 -c bgp/test_bgp_fact.py -f vtestbed.csv -i veos_vtb
 ```
 
+If neighbor devices are SONiC
+
+```
+./run_tests.sh -n vms-kvm-t0 -d vlab-01 -c bgp/test_bgp_fact.py -f vtestbed.csv -i veos_vtb -e "--neighbor_type=sonic"
+```
+
 You should see three sets of tests run and pass. You're now set up and ready to use the KVM testbed!

tests/common/devices/sonic.py

@@ -32,7 +32,8 @@ class SonicHost(AnsibleHostBase):
 
 
     def __init__(self, ansible_adhoc, hostname,
-                 shell_user=None, shell_passwd=None):
+                 shell_user=None, shell_passwd=None,
+                 ssh_user=None, ssh_passwd=None):
         AnsibleHostBase.__init__(self, ansible_adhoc, hostname)
 
         if shell_user and shell_passwd:
@@ -58,6 +59,13 @@ def __init__(self, ansible_adhoc, hostname,
                 if pass_var in hostvars:
                     vm.extra_vars.update({pass_var: shell_passwd})
 
+        if ssh_user and ssh_passwd:
+            evars = {
+                'ansible_ssh_user': ssh_user,
+                'ansible_ssh_pass': ssh_passwd,
+            }
+            self.host.options['variable_manager'].extra_vars.update(evars)
+
         self._facts = self._gather_facts()
         self._os_version = self._get_os_version()
         self.is_multi_asic = True if self.facts["num_asic"] > 1 else False
@@ -152,10 +160,31 @@ def _gather_facts(self):
         facts["num_asic"] = self._get_asic_count(facts["platform"])
         facts["router_mac"] = self._get_router_mac()
         facts["modular_chassis"] = self._get_modular_chassis()
+        facts["mgmt_interface"] = self._get_mgmt_interface()
 
         logging.debug("Gathered SonicHost facts: %s" % json.dumps(facts))
         return facts
 
+    def _get_mgmt_interface(self):
+        """
+        Gets the IPs of management interface
+        Output example
+
+            admin@ARISTA04T1:~$ show management_interface address
+            Management IP address = 10.250.0.54/24
+            Management Network Default Gateway = 10.250.0.1
+            Management IP address = 10.250.0.59/24
+            Management Network Default Gateway = 10.250.0.1
+
+        """
+        show_cmd_output = self.shell("show management_interface address", module_ignore_errors=True)
+        mgmt_addrs = []
+        for line in show_cmd_output["stdout_lines"]:
+            addr = re.match(r"Management IP address = (\d{0,3}\.\d{0,3}\.\d{0,3}\.\d{0,3})\/\d+", line)
+            if addr:
+                mgmt_addrs.append(addr.group(1))
+        return mgmt_addrs
+
     def _get_modular_chassis(self):
         py_res = self.shell("python -c \"import sonic_platform\"", module_ignore_errors=True)
         if py_res["failed"]:

tests/conftest.py

@@ -15,6 +15,7 @@
 from tests.common.devices.local import Localhost
 from tests.common.devices.ptf import PTFHost
 from tests.common.devices.eos import EosHost
+from tests.common.devices.sonic import SonicHost
 from tests.common.devices.fanout import FanoutHost
 from tests.common.devices.k8s import K8sMasterHost
 from tests.common.devices.k8s import K8sMasterCluster
@@ -66,6 +67,10 @@ def pytest_addoption(parser):
     # Kubernetes master options
     parser.addoption("--kube_master", action="store", default=None, type=str, help="Name of k8s master group used in k8s inventory, format: k8s_vms{msetnumber}_{servernumber}")
 
+    # neighbor device type
+    parser.addoption("--neighbor_type", action="store", default="eos", type=str, choices=["eos", "sonic"],
+                    help="Neighbor devices type")
+
     ############################
     # pfc_asym options         #
     ############################
@@ -346,21 +351,31 @@ def k8scluster(k8smasters):
     return k8s_master_cluster
 
 @pytest.fixture(scope="module")
-def nbrhosts(ansible_adhoc, tbinfo, creds):
+def nbrhosts(ansible_adhoc, tbinfo, creds, request):
     """
     Shortcut fixture for getting VM host
     """
 
     vm_base = int(tbinfo['vm_base'][2:])
+    neighbor_type = request.config.getoption("--neighbor_type")
     devices = {}
     for k, v in tbinfo['topo']['properties']['topology']['VMs'].items():
-        devices[k] = {'host': EosHost(ansible_adhoc,
-                                      "VM%04d" % (vm_base + v['vm_offset']),
-                                      creds['eos_login'],
-                                      creds['eos_password'],
-                                      shell_user=creds['eos_root_user'] if 'eos_root_user' in creds else None,
-                                      shell_passwd=creds['eos_root_password'] if 'eos_root_password' in creds else None),
-                      'conf': tbinfo['topo']['properties']['configuration'][k]}
+        if neighbor_type == "eos":
+            devices[k] = {'host': EosHost(ansible_adhoc,
+                                        "VM%04d" % (vm_base + v['vm_offset']),
+                                        creds['eos_login'],
+                                        creds['eos_password'],
+                                        shell_user=creds['eos_root_user'] if 'eos_root_user' in creds else None,
+                                        shell_passwd=creds['eos_root_password'] if 'eos_root_password' in creds else None),
+                        'conf': tbinfo['topo']['properties']['configuration'][k]}
+        elif neighbor_type == "sonic":
+            devices[k] = {'host': SonicHost(ansible_adhoc,
+                                        "VM%04d" % (vm_base + v['vm_offset']),
+                                        ssh_user=creds['sonic_login'] if 'sonic_login' in creds else None,
+                                        ssh_passwd=creds['sonic_password'] if 'sonic_password' in creds else None),
+                        'conf': tbinfo['topo']['properties']['configuration'][k]}
+        else:
+            raise ValueError("Unknown neighbor type %s" % (neighbor_type, ))
     return devices
 
 @pytest.fixture(scope="module")
@@ -434,6 +449,14 @@ def eos():
         return eos
 
 
+@pytest.fixture(scope='session')
+def sonic():
+    """ read and yield sonic configuration """
+    with open('sonic/sonic.yml') as stream:
+        eos = yaml.safe_load(stream)
+        return eos
+
+
 @pytest.fixture(scope='session')
 def pdu():
     """ read and yield pdu configuration """

tests/sonic/sonic.yml

@@ -0,0 +1,2 @@
+# snmp variables
+snmp_rocommunity: public

tests/test_nbr_health.py

@@ -3,6 +3,8 @@
 import logging
 
 from common.helpers.assertions import pytest_assert
+from common.devices.eos import EosHost
+from common.devices.sonic import SonicHost
 
 logger = logging.getLogger(__name__)
 
@@ -13,9 +15,9 @@
     pytest.mark.topology('util') #special marker
 ]
 
-def check_snmp(hostname, mgmt_addr, localhost, community):
+def check_snmp(hostname, mgmt_addr, localhost, community, is_eos):
     logger.info("Check neighbor {}, mgmt ip {} snmp".format(hostname, mgmt_addr))
-    res = localhost.snmp_facts(host=mgmt_addr, version='v2c', is_eos=True, community=community)
+    res = localhost.snmp_facts(host=mgmt_addr, version='v2c', is_eos=is_eos, community=community)
     try:
         snmp_data = res['ansible_facts']
     except:
@@ -45,14 +47,33 @@ def check_eos_facts(hostname, mgmt_addr, host):
 
     return "neighbor {} has no management address {}".format(hostname, mgmt_ip)
 
-def check_bgp_facts(hostname, host):
+def check_sonic_facts(hostname, mgmt_addr, host):
+    logger.info("Check neighbor {} eos facts".format(hostname))
+    res = host.facts
+    logger.info("facts: {}".format(json.dumps(res, indent=4)))
+    mgmt_addrs = host.facts['mgmt_interface']
+    if len(mgmt_addrs) == 0:
+        return "there is no management interface in neighbor {}".format(hostname)
+    for addr in mgmt_addrs:
+        if addr == mgmt_addr:
+            return
+    return "neighbor {} has no management address {}".format(hostname, mgmt_ip)
+
+def check_eos_bgp_facts(hostname, host):
     logger.info("Check neighbor {} bgp facts".format(hostname))
     res = host.eos_command(commands=['show ip bgp sum'])
     logger.info("bgp: {}".format(res))
     if not res.has_key('stdout_lines') or u'BGP summary' not in res['stdout_lines'][0][0]:
         return "neighbor {} bgp not configured correctly".format(hostname)
 
-def test_neighbors_health(duthosts, localhost, nbrhosts, eos, enum_frontend_dut_hostname):
+def check_sonic_bgp_facts(hostname, host):
+    logger.info("Check neighbor {} bgp facts".format(hostname))
+    res = host.command('vtysh -c "show ip bgp sum"')
+    logger.info("bgp: {}".format(res))
+    if not res.has_key('stdout_lines') or u'Unicast Summary' not in "\n".join(res['stdout_lines']):
+        return "neighbor {} bgp not configured correctly".format(hostname)
+
+def test_neighbors_health(duthosts, localhost, nbrhosts, eos, sonic, enum_frontend_dut_hostname):
     """Check each neighbor device health"""
 
     fails = []
@@ -74,19 +95,39 @@ def test_neighbors_health(duthosts, localhost, nbrhosts, eos, enum_frontend_dut_
             # The server neighbors need to be skipped too.
             continue
 
-        failmsg = check_snmp(k, v['mgmt_addr'], localhost, eos['snmp_rocommunity'])
-        if failmsg:
-            fails.append(failmsg)
+        nbrhost = nbrhosts[k]['host']
 
-        eoshost = nbrhosts[k]['host']
-        failmsg = check_eos_facts(k, v['mgmt_addr'], eoshost)
-        if failmsg:
-            fails.append(failmsg)
+        if isinstance(nbrhost, EosHost):
+            failmsg = check_snmp(k, v['mgmt_addr'], localhost, eos['snmp_rocommunity'], True)
+            if failmsg:
+                fails.append(failmsg)
+
+            failmsg = check_eos_facts(k, v['mgmt_addr'], nbrhost)
+            if failmsg:
+                fails.append(failmsg)
+
+            failmsg = check_eos_bgp_facts(k, nbrhost)
+            if failmsg:
+                fails.append(failmsg)
 
-        failmsg = check_bgp_facts(k, eoshost)
-        if failmsg:
+        elif isinstance(nbrhost, SonicHost):
+            failmsg = check_snmp(k, v['mgmt_addr'], localhost, sonic['snmp_rocommunity'], False)
+            if failmsg:
+                fails.append(failmsg)
+
+            failmsg = check_sonic_facts(k, v['mgmt_addr'], nbrhost)
+            if failmsg:
+                fails.append(failmsg)
+
+            failmsg = check_sonic_bgp_facts(k, nbrhost)
+            if failmsg:
+                fails.append(failmsg)
+
+        else:
+            failmsg = "neighbor type {} is unknown".format(k)
             fails.append(failmsg)
 
+
     # TODO: check link, bgp, etc. on
 
     pytest_assert(len(fails) == 0, "\n".join(fails))