diff --git a/scripts/route_check.py b/scripts/route_check.py index 3cdcf0d320..db5fce8f3c 100755 --- a/scripts/route_check.py +++ b/scripts/route_check.py @@ -676,9 +676,10 @@ def check_frr_pending_routes(namespace): retries = FRR_CHECK_RETRIES for i in range(retries): missed_rt = [] + failed_rt = [] frr_routes = get_frr_routes_parallel(namespace) - for _, entries in frr_routes.items(): + for route_prefix, entries in frr_routes.items(): for entry in entries: if entry['protocol'] in ('connected', 'kernel', 'static'): continue @@ -695,12 +696,16 @@ def check_frr_pending_routes(namespace): if not entry.get('offloaded', False): missed_rt.append(entry) - if not missed_rt: + if entry.get('failed', False): + failed_rt.append(route_prefix) + + if not missed_rt and not failed_rt: break time.sleep(FRR_WAIT_TIME) - print_message(syslog.LOG_DEBUG, "FRR missed routes: {}".format(missed_rt, indent=4)) - return missed_rt + print_message(syslog.LOG_DEBUG, "FRR missed routes: {}".format(json.dumps(missed_rt, indent=4))) + print_message(syslog.LOG_DEBUG, "FRR failed routes: {}".format(json.dumps(failed_rt, indent=4))) + return missed_rt, failed_rt def mitigate_installed_not_offloaded_frr_routes(namespace, missed_frr_rt, rt_appl): @@ -837,6 +842,7 @@ def check_routes_for_namespace(namespace): rt_appl_miss = [] rt_asic_miss = [] rt_frr_miss = [] + rt_frr_failed = [] selector, subs, rt_asic = get_asicdb_routes(namespace) @@ -890,17 +896,23 @@ def check_routes_for_namespace(namespace): if rt_asic_miss: results["Unaccounted_ROUTE_ENTRY_TABLE_entries"] = rt_asic_miss - rt_frr_miss = check_frr_pending_routes(namespace) + rt_frr_miss, rt_frr_failed = check_frr_pending_routes(namespace) if rt_frr_miss: results["missed_FRR_routes"] = rt_frr_miss + if rt_frr_failed: + results["failed_FRR_routes"] = rt_frr_failed + if results: if rt_frr_miss and not rt_appl_miss and not rt_asic_miss: print_message(syslog.LOG_ERR, "Some routes are not set offloaded in FRR{} \ but all routes in APPL_DB and ASIC_DB are in sync".format(namespace)) if is_suppress_fib_pending_enabled(namespace): mitigate_installed_not_offloaded_frr_routes(namespace, rt_frr_miss, rt_appl) + if rt_frr_failed: + print_message(syslog.LOG_ERR, "Some routes have failed state in FRR {} \ + : {}".format(namespace, rt_frr_failed)) return results, adds, deletes diff --git a/tests/route_check_test_data.py b/tests/route_check_test_data.py index f92d7fd4ef..6db84d5f3d 100644 --- a/tests/route_check_test_data.py +++ b/tests/route_check_test_data.py @@ -1354,4 +1354,354 @@ } } }, + "29": { + DESCR: "failure test case, failed FRR routes", + MULTI_ASIC: False, + NAMESPACE: [''], + ARGS: "route_check -m INFO -i 1000", + PRE: { + DEFAULTNS: { + APPL_DB: { + ROUTE_TABLE: { + "0.0.0.0/0": {"ifname": "portchannel0"}, + "10.10.196.12/31": {"ifname": "portchannel0"}, + "10.10.196.20/31": {"ifname": "portchannel0"}, + }, + INTF_TABLE: { + "PortChannel1013:10.10.196.24/31": {}, + "PortChannel1023:2603:10b0:503:df4::5d/126": {}, + "PortChannel1024": {} + } + }, + ASIC_DB: { + RT_ENTRY_TABLE: { + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {} + } + }, + }, + }, + FRR_ROUTES: { + DEFAULTNS: { + "0.0.0.0/0": [ + { + "prefix": "0.0.0.0/0", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + }, + ], + "10.10.196.12/31": [ + { + "prefix": "10.10.196.12/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.20/31": [ + { + "prefix": "10.10.196.20/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.24/31": [ + { + "protocol": "connected", + "selected": True, + }, + ], + }, + }, + RESULT: { + DEFAULTNS: { + "failed_FRR_routes": [ + "10.10.196.12/31", + "10.10.196.20/31" + ], + }, + }, + RET: -1, + }, + "30": { + DESCR: "multi-asic failure test case, failed FRR routes", + MULTI_ASIC: True, + NAMESPACE: ['asic0', 'asic1'], + ARGS: "route_check -n asic0 -m INFO -i 1000", + PRE: { + ASIC0: { + APPL_DB: { + ROUTE_TABLE: { + "0.0.0.0/0": {"ifname": "portchannel0"}, + "10.10.196.12/31": {"ifname": "portchannel0"}, + "10.10.196.20/31": {"ifname": "portchannel0"}, + }, + INTF_TABLE: { + "PortChannel1013:10.10.196.24/31": {}, + "PortChannel1023:2603:10b0:503:df4::5d/126": {}, + "PortChannel1024": {} + } + }, + ASIC_DB: { + RT_ENTRY_TABLE: { + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {} + } + }, + }, + }, + FRR_ROUTES: { + ASIC0: { + "0.0.0.0/0": [ + { + "prefix": "0.0.0.0/0", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + }, + ], + "10.10.196.12/31": [ + { + "prefix": "10.10.196.12/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.20/31": [ + { + "prefix": "10.10.196.20/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": False, + "failed": True, + }, + ], + "10.10.196.24/31": [ + { + "protocol": "connected", + "selected": True, + }, + ], + }, + }, + RESULT: { + ASIC0: { + "missed_FRR_routes": [ + {"prefix": "10.10.196.20/31", "vrfName": "default", "protocol": "bgp", + "selected": True, "offloaded": False, "failed": True} + ], + "failed_FRR_routes": [ + "10.10.196.12/31", + "10.10.196.20/31" + ], + }, + }, + RET: -1, + }, + "31": { + DESCR: "missed and failed FRR routes with APPL_DB and ASIC_DB in sync (mitigation case)", + MULTI_ASIC: False, + NAMESPACE: [''], + ARGS: "route_check -m INFO -i 1000", + PRE: { + DEFAULTNS: { + APPL_DB: { + ROUTE_TABLE: { + "0.0.0.0/0": {"ifname": "portchannel0"}, + "10.10.196.12/31": {"ifname": "portchannel0"}, + "10.10.196.20/31": {"ifname": "portchannel0"}, + "192.168.1.0/24": {"ifname": "portchannel0"}, + }, + INTF_TABLE: { + "PortChannel1013:10.10.196.24/31": {}, + "PortChannel1023:2603:10b0:503:df4::5d/126": {}, + "PortChannel1024": {} + } + }, + ASIC_DB: { + RT_ENTRY_TABLE: { + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "192.168.1.0/24" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {} + } + }, + }, + }, + FRR_ROUTES: { + DEFAULTNS: { + "0.0.0.0/0": [ + { + "prefix": "0.0.0.0/0", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + }, + ], + "10.10.196.12/31": [ + { + "prefix": "10.10.196.12/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": False, # Missing offload flag + }, + ], + "10.10.196.20/31": [ + { + "prefix": "10.10.196.20/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, # Failed route + }, + ], + "192.168.1.0/24": [ + { + "prefix": "192.168.1.0/24", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": False, # Missing offload flag + "failed": True, # Failed route + }, + ], + "10.10.196.24/31": [ + { + "protocol": "connected", + "selected": True, + }, + ], + }, + }, + RESULT: { + DEFAULTNS: { + "missed_FRR_routes": [ + {"prefix": "10.10.196.12/31", "vrfName": "default", "protocol": "bgp", + "selected": True, "offloaded": False}, + {"prefix": "192.168.1.0/24", "vrfName": "default", "protocol": "bgp", + "selected": True, "offloaded": False, "failed": True} + ], + "failed_FRR_routes": [ + "10.10.196.20/31", + "192.168.1.0/24" + ], + }, + }, + RET: -1, + }, + "32": { + DESCR: "failure test case, only failed FRR routes (no missing routes)", + MULTI_ASIC: False, + NAMESPACE: [''], + ARGS: "route_check -m INFO -i 1000", + PRE: { + DEFAULTNS: { + APPL_DB: { + ROUTE_TABLE: { + "0.0.0.0/0": {"ifname": "portchannel0"}, + "10.10.196.12/31": {"ifname": "portchannel0"}, + "10.10.196.20/31": {"ifname": "portchannel0"}, + "192.168.1.0/24": {"ifname": "portchannel0"}, + }, + INTF_TABLE: { + "PortChannel1013:10.10.196.24/31": {}, + "PortChannel1023:2603:10b0:503:df4::5d/126": {}, + "PortChannel1024": {} + } + }, + ASIC_DB: { + RT_ENTRY_TABLE: { + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.12/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.20/31" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "192.168.1.0/24" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "10.10.196.24/32" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "2603:10b0:503:df4::5d/128" + ASIC_RT_ENTRY_KEY_SUFFIX: {}, + ASIC_RT_ENTRY_KEY_PREFIX + "0.0.0.0/0" + ASIC_RT_ENTRY_KEY_SUFFIX: {} + } + }, + }, + }, + FRR_ROUTES: { + DEFAULTNS: { + "0.0.0.0/0": [ + { + "prefix": "0.0.0.0/0", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + }, + ], + "10.10.196.12/31": [ + { + "prefix": "10.10.196.12/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.20/31": [ + { + "prefix": "10.10.196.20/31", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "192.168.1.0/24": [ + { + "prefix": "192.168.1.0/24", + "vrfName": "default", + "protocol": "bgp", + "selected": True, + "offloaded": True, + "failed": True, + }, + ], + "10.10.196.24/31": [ + { + "protocol": "connected", + "selected": True, + }, + ], + }, + }, + RESULT: { + DEFAULTNS: { + "failed_FRR_routes": [ + "10.10.196.12/31", + "10.10.196.20/31", + "192.168.1.0/24" + ], + }, + }, + RET: -1, + }, }