diff --git a/scripts/route_check.py b/scripts/route_check.py index c2cb4f721..0d6406913 100755 --- a/scripts/route_check.py +++ b/scripts/route_check.py @@ -599,12 +599,14 @@ def check_frr_pending_routes(namespace): """ missed_rt = [] + failed_rt = [] retries = FRR_CHECK_RETRIES for i in range(retries): missed_rt = [] + failed_rt = [] frr_routes = get_frr_routes(namespace) - for _, entries in frr_routes.items(): + for route_prefix, entries in frr_routes.items(): for entry in entries: if entry['protocol'] in ('connected', 'kernel', 'static'): continue @@ -621,12 +623,16 @@ def check_frr_pending_routes(namespace): if not entry.get('offloaded', False): missed_rt.append(entry) - if not missed_rt: + if entry.get('failed', False): + failed_rt.append(route_prefix) + + if not missed_rt and not failed_rt: break time.sleep(FRR_WAIT_TIME) - print_message(syslog.LOG_DEBUG, "FRR missed routes: {}".format(missed_rt, indent=4)) - return missed_rt + print_message(syslog.LOG_DEBUG, "FRR missed routes: {}".format(json.dumps(missed_rt, indent=4))) + print_message(syslog.LOG_DEBUG, "FRR failed routes: {}".format(json.dumps(failed_rt, indent=4))) + return missed_rt, failed_rt def mitigate_installed_not_offloaded_frr_routes(namespace, missed_frr_rt, rt_appl): @@ -769,6 +775,7 @@ def check_routes(namespace): rt_appl_miss = [] rt_asic_miss = [] rt_frr_miss = [] + rt_frr_failed = [] adds[namespace] = [] deletes[namespace] = [] @@ -826,19 +833,27 @@ def check_routes(namespace): results[namespace] = {} results[namespace]["Unaccounted_ROUTE_ENTRY_TABLE_entries"] = rt_asic_miss - rt_frr_miss = check_frr_pending_routes(namespace) + rt_frr_miss, rt_frr_failed = check_frr_pending_routes(namespace) if rt_frr_miss: if namespace not in results: results[namespace] = {} results[namespace]["missed_FRR_routes"] = rt_frr_miss + if rt_frr_failed: + if namespace not in results: + results[namespace] = {} + results[namespace]["failed_FRR_routes"] = rt_frr_failed + if results: if rt_frr_miss and not rt_appl_miss and not rt_asic_miss: print_message(syslog.LOG_ERR, "Some routes are not set offloaded in FRR{} \ but all routes in APPL_DB and ASIC_DB are in sync".format(namespace)) if is_suppress_fib_pending_enabled(namespace): mitigate_installed_not_offloaded_frr_routes(namespace, rt_frr_miss, rt_appl) + if rt_frr_failed: + print_message(syslog.LOG_ERR, "Some routes have failed state in FRR {} \ + : {}".format(namespace, rt_frr_failed)) if results: print_message(syslog.LOG_WARNING, "Failure results: {", json.dumps(results, indent=4), "}") diff --git a/tests/route_check_test_data.py b/tests/route_check_test_data.py index b33edd6dc..2265f4a27 100644 --- a/tests/route_check_test_data.py +++ b/tests/route_check_test_data.py @@ -1354,4 +1354,354 @@ } } }, + "35": { + DESCR: "failure test case, failed FRR routes", + MULTI_ASIC: False, + NAMESPACE: [''], + ARGS: "route_check -m INFO -i 1000", + PRE: { + DEFAULTNS: { + APPL_DB: { + ROUTE_TABLE: { + "0.0.0.0/0": {"ifname": "portchannel0"}, + "10.10.196.12/31": {"ifname": "portchannel0"}, + "10.10.196.20/31": {"ifname": "portchannel0"}, + }, + INTF_TABLE: { + "PortChannel1013:10.10.196.24/31": {}, + "PortChannel1023:2603:10b0:503:df4::5d/126": {}, + "PortChannel1024": {} + } + }, + ASIC_DB: { + RT_ENTRY_TABLE: { + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {} + } + }, + }, + }, + FRR_ROUTES: { + DEFAULTNS: { + "0.0.0.0/0": [ + { + "prefix": "0.0.0.0/0", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + }, + ], + "10.10.196.12/31": [ + { + "prefix": "10.10.196.12/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.20/31": [ + { + "prefix": "10.10.196.20/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.24/31": [ + { + "protocol": "connected", + "selected": True, + }, + ], + }, + }, + RESULT: { + DEFAULTNS: { + "failed_FRR_routes": [ + "10.10.196.12/31", + "10.10.196.20/31" + ], + }, + }, + RET: -1, + }, + "36": { + DESCR: "multi-asic failure test case, failed FRR routes", + MULTI_ASIC: True, + NAMESPACE: ['asic0', 'asic1'], + ARGS: "route_check -n asic0 -m INFO -i 1000", + PRE: { + ASIC0: { + APPL_DB: { + ROUTE_TABLE: { + "0.0.0.0/0": {"ifname": "portchannel0"}, + "10.10.196.12/31": {"ifname": "portchannel0"}, + "10.10.196.20/31": {"ifname": "portchannel0"}, + }, + INTF_TABLE: { + "PortChannel1013:10.10.196.24/31": {}, + "PortChannel1023:2603:10b0:503:df4::5d/126": {}, + "PortChannel1024": {} + } + }, + ASIC_DB: { + RT_ENTRY_TABLE: { + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {} + } + }, + }, + }, + FRR_ROUTES: { + ASIC0: { + "0.0.0.0/0": [ + { + "prefix": "0.0.0.0/0", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + }, + ], + "10.10.196.12/31": [ + { + "prefix": "10.10.196.12/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.20/31": [ + { + "prefix": "10.10.196.20/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": False, + "failed": True, + }, + ], + "10.10.196.24/31": [ + { + "protocol": "connected", + "selected": True, + }, + ], + }, + }, + RESULT: { + ASIC0: { + "missed_FRR_routes": [ + {"prefix": "10.10.196.20/31", "vrfName": "default", "protocol": "bgp", + "selected": True, "offloaded": False, "failed": True} + ], + "failed_FRR_routes": [ + "10.10.196.12/31", + "10.10.196.20/31" + ], + }, + }, + RET: -1, + }, + "37": { + DESCR: "missed and failed FRR routes with APPL_DB and ASIC_DB in sync (mitigation case)", + MULTI_ASIC: False, + NAMESPACE: [''], + ARGS: "route_check -m INFO -i 1000", + PRE: { + DEFAULTNS: { + APPL_DB: { + ROUTE_TABLE: { + "0.0.0.0/0": {"ifname": "portchannel0"}, + "10.10.196.12/31": {"ifname": "portchannel0"}, + "10.10.196.20/31": {"ifname": "portchannel0"}, + "192.168.1.0/24": {"ifname": "portchannel0"}, + }, + INTF_TABLE: { + "PortChannel1013:10.10.196.24/31": {}, + "PortChannel1023:2603:10b0:503:df4::5d/126": {}, + "PortChannel1024": {} + } + }, + ASIC_DB: { + RT_ENTRY_TABLE: { + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "192.168.1.0/24" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {} + } + }, + }, + }, + FRR_ROUTES: { + DEFAULTNS: { + "0.0.0.0/0": [ + { + "prefix": "0.0.0.0/0", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + }, + ], + "10.10.196.12/31": [ + { + "prefix": "10.10.196.12/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": False, + }, + ], + "10.10.196.20/31": [ + { + "prefix": "10.10.196.20/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "192.168.1.0/24": [ + { + "prefix": "192.168.1.0/24", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": False, + "failed": True, + }, + ], + "10.10.196.24/31": [ + { + "protocol": "connected", + "selected": True, + }, + ], + }, + }, + RESULT: { + DEFAULTNS: { + "missed_FRR_routes": [ + {"prefix": "10.10.196.12/31", "vrfName": "default", "protocol": "bgp", + "selected": True, "offloaded": False}, + {"prefix": "192.168.1.0/24", "vrfName": "default", "protocol": "bgp", + "selected": True, "offloaded": False, "failed": True} + ], + "failed_FRR_routes": [ + "10.10.196.20/31", + "192.168.1.0/24" + ], + }, + }, + RET: -1, + }, + "38": { + DESCR: "failure test case, only failed FRR routes (no missing routes)", + MULTI_ASIC: False, + NAMESPACE: [''], + ARGS: "route_check -m INFO -i 1000", + PRE: { + DEFAULTNS: { + APPL_DB: { + ROUTE_TABLE: { + "0.0.0.0/0": {"ifname": "portchannel0"}, + "10.10.196.12/31": {"ifname": "portchannel0"}, + "10.10.196.20/31": {"ifname": "portchannel0"}, + "192.168.1.0/24": {"ifname": "portchannel0"}, + }, + INTF_TABLE: { + "PortChannel1013:10.10.196.24/31": {}, + "PortChannel1023:2603:10b0:503:df4::5d/126": {}, + "PortChannel1024": {} + } + }, + ASIC_DB: { + RT_ENTRY_TABLE: { + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "192.168.1.0/24" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {} + } + }, + }, + }, + FRR_ROUTES: { + DEFAULTNS: { + "0.0.0.0/0": [ + { + "prefix": "0.0.0.0/0", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + }, + ], + "10.10.196.12/31": [ + { + "prefix": "10.10.196.12/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.20/31": [ + { + "prefix": "10.10.196.20/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "192.168.1.0/24": [ + { + "prefix": "192.168.1.0/24", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.24/31": [ + { + "protocol": "connected", + "selected": True, + }, + ], + }, + }, + RESULT: { + DEFAULTNS: { + "failed_FRR_routes": [ + "10.10.196.12/31", + "10.10.196.20/31", + "192.168.1.0/24" + ], + }, + }, + RET: -1, + }, }