[sanity_check][bgp] Add default route check in sanity

Author: yaqiangz

tests/common/plugins/sanity_check/checks.py

@@ -166,6 +166,12 @@ def _check_bgp_on_dut(*args, **kwargs):
         dut = kwargs['node']
         results = kwargs['results']
 
+        def _check_default_route(version, asichost):
+            # Return True if successfully get default route
+            res = asichost.shell("ip {} route show default".format("" if version == 4 else "-6"),
+                                 module_ignore_errors=True)
+            return not res["rc"] and len(res["stdout"].strip()) != 0
+
         def _check_bgp_status_helper():
             asic_check_results = []
             bgp_facts = dut.bgp_facts(asic_index='all')
@@ -187,23 +193,32 @@ def _check_bgp_status_helper():
                 if a_asic_neighbors is not None and len(a_asic_neighbors) > 0:
                     down_neighbors = [k for k, v in list(a_asic_neighbors.items())
                                       if v['state'] != 'established']
+                    asic_key = 'bgp' if dut.facts['num_asic'] == 1 else 'bgp' + str(asic_index)
                     if down_neighbors:
-                        if dut.facts['num_asic'] == 1:
-                            check_result['bgp'] = {'down_neighbors': down_neighbors}
-                        else:
-                            check_result['bgp' + str(asic_index)] = {'down_neighbors': down_neighbors}
+                        check_result[asic_key] = {'down_neighbors': down_neighbors}
                         a_asic_result = True
                     else:
                         a_asic_result = False
-                        if dut.facts['num_asic'] == 1:
-                            if 'bgp' in check_result:
-                                check_result['bgp'].pop('down_neighbors', None)
-                        else:
-                            if 'bgp' + str(asic_index) in check_result:
-                                check_result['bgp' + str(asic_index)].pop('down_neighbors', None)
+                        if asic_key in check_result:
+                            check_result[asic_key].pop('down_neighbors', None)
                 else:
                     a_asic_result = True
 
+                # Add default route check for default route missing issue in below scenario:
+                # 1) Loopbackv4 ip address replace, it would cause all bgp routes missing
+                # 2) Announce or withdraw routes in some test cases and doesn't recover it
+                asic_host = dut.asic_instance(asic_index)
+                if not _check_default_route(4, asic_host):
+                    if asic_key not in check_result:
+                        check_result[asic_key] = {}
+                    check_result[asic_key]["no_v4_default_route"] = True
+                    a_asic_result = True
+                if not _check_default_route(6, asic_host):
+                    if asic_key not in check_result:
+                        check_result[asic_key] = {}
+                    check_result[asic_key]["no_v6_default_route"] = True
+                    a_asic_result = True
+
                 asic_check_results.append(a_asic_result)
 
             if any(asic_check_results):
@@ -243,8 +258,12 @@ def _check_bgp_status_helper():
                     if 'down_neighbors' in check_result[a_result]:
                         logger.info('BGP neighbors down: %s on bgp instance %s on dut %s' % (
                             check_result[a_result]['down_neighbors'], a_result, dut.hostname))
+                    if "no_v4_default_route" in check_result[a_result]:
+                        logger.info('Deafult v4 route for {} {} is missing'.format(dut.hostname, a_result))
+                    if "no_v6_default_route" in check_result[a_result]:
+                        logger.info('Deafult v6 route for {} {} is missing'.format(dut.hostname, a_result))
         else:
-            logger.info('No BGP neighbors are down on %s' % dut.hostname)
+            logger.info('No BGP neighbors are down or default route missing on %s' % dut.hostname)
 
         mgFacts = dut.get_extended_minigraph_facts(tbinfo)
         if dut.num_asics() == 1 and tbinfo['topo']['type'] != 't2' and \

tests/common/plugins/sanity_check/recover.py

@@ -147,6 +147,12 @@ def _recover_with_command(dut, cmd, wait_time):
     wait(wait_time, msg="Wait {} seconds for system to be stable.".format(wait_time))
 
 
+def re_announce_routes(localhost, topo_name, ptf_ip):
+    localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="withdraw", path="../ansible/")
+    localhost.announce_routes(topo_name=topo_name, ptf_ip=ptf_ip, action="announce", path="../ansible/")
+    return None
+
+
 def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_results, wait_time):
     outstanding_action = None
     for result in check_results:
@@ -155,7 +161,16 @@ def adaptive_recover(dut, localhost, fanouthosts, nbrhosts, tbinfo, check_result
                 action = _recover_interfaces(dut, fanouthosts, result, wait_time)
             elif result['check_item'] == 'services':
                 action = _recover_services(dut, result)
-            elif result['check_item'] == 'bgp' or result['check_item'] == "neighbor_macsec_empty":
+            elif result['check_item'] == 'bgp':
+                # If there is only default route missing issue, only need to re-announce routes to recover
+                if ("no_v4_default_route" in result['bgp'] and len(result['bgp']) == 1 or
+                    "no_v6_default_route" in result['bgp'] and len(result['bgp']) == 1 or
+                    ("no_v4_default_route" in result['bgp'] and "no_v6_default_route" in result['bgp'] and
+                     len(result['bgp']) == 2)):
+                    action = re_announce_routes(localhost, tbinfo["topo"]["name"], tbinfo["ptf_ip"])
+                else:
+                    action = neighbor_vm_restore(dut, nbrhosts, tbinfo, result)
+            elif result['check_item'] == "neighbor_macsec_empty":
                 action = neighbor_vm_restore(dut, nbrhosts, tbinfo, result)
             elif result['check_item'] in ['processes', 'mux_simulator']:
                 action = 'config_reload'