diff --git a/bgpd/bgp_fsm.c b/bgpd/bgp_fsm.c index d41ef8abbac9..76624fee9e43 100644 --- a/bgpd/bgp_fsm.c +++ b/bgpd/bgp_fsm.c @@ -2046,9 +2046,10 @@ static int bgp_start_deferral_timer(struct bgp *bgp, afi_t afi, safi_t safi, } gr_info->eor_required++; /* Send message to RIB indicating route update pending */ - if (gr_info->af_enabled[afi][safi] == false) { - gr_info->af_enabled[afi][safi] = true; - /* Send message to RIB */ + if (gr_info->af_enabled == false) { + gr_info->af_enabled = true; + gr_info->route_sync = false; + bgp->gr_route_sync_pending = true; bgp_zebra_update(bgp, afi, safi, ZEBRA_CLIENT_ROUTE_UPDATE_PENDING); } @@ -2082,7 +2083,7 @@ static int bgp_update_gr_info(struct peer *peer, afi_t afi, safi_t safi) if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer) && BGP_PEER_RESTARTING_MODE(peer)) { /* Check if the forwarding state is preserved */ - if (CHECK_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD)) { + if (bgp_gr_is_forwarding_preserved(bgp)) { gr_info = &(bgp->gr_info[afi][safi]); ret = bgp_start_deferral_timer(bgp, afi, safi, gr_info); } @@ -2199,8 +2200,7 @@ bgp_establish(struct peer_connection *connection) } else { if (BGP_PEER_GRACEFUL_RESTART_CAPABLE(peer) && BGP_PEER_RESTARTING_MODE(peer) && - CHECK_FLAG(peer->bgp->flags, - BGP_FLAG_GR_PRESERVE_FWD)) + bgp_gr_is_forwarding_preserved(peer->bgp)) peer->bgp->gr_info[afi][safi] .eor_required++; } @@ -2695,53 +2695,60 @@ int bgp_event_update(struct peer_connection *connection, } /* BGP GR Code */ -int bgp_gr_lookup_n_update_all_peer(struct bgp *bgp, - enum global_mode global_new_state, - enum global_mode global_old_state) +static inline void +bgp_peer_inherit_global_gr_mode(struct peer *peer, + enum global_mode global_gr_mode) +{ + switch (global_gr_mode) { + case GLOBAL_HELPER: + BGP_PEER_GR_HELPER_ENABLE(peer); + break; + case GLOBAL_GR: + BGP_PEER_GR_ENABLE(peer); + break; + case GLOBAL_DISABLE: + BGP_PEER_GR_DISABLE(peer); + break; + case GLOBAL_INVALID: + default: + zlog_err("Unexpected Global GR mode %d", global_gr_mode); + } +} + +static void bgp_gr_update_mode_of_all_peers(struct bgp *bgp, + enum global_mode global_new_state) { struct peer *peer = {0}; struct listnode *node = {0}; struct listnode *nnode = {0}; enum peer_mode peer_old_state = PEER_INVALID; - for (ALL_LIST_ELEMENTS(bgp->peer, node, nnode, peer)) { - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("%s [BGP_GR] Peer: (%s) :", __func__, - peer->host); + /* TODO: Need to handle peer-groups. */ + for (ALL_LIST_ELEMENTS(bgp->peer, node, nnode, peer)) { peer_old_state = bgp_peer_gr_mode_get(peer); + if (peer_old_state != PEER_GLOBAL_INHERIT) + continue; - if (peer_old_state == PEER_GLOBAL_INHERIT) { + bgp_peer_inherit_global_gr_mode(peer, global_new_state); + bgp_peer_gr_flags_update(peer); - /* - *Reset only these peers and send a - *new open message with the change capabilities. - *Considering the mode to be "global_new_state" and - *do all operation accordingly - */ + if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) + zlog_debug("%pBP: Inherited Global GR mode, GR flags 0x%x peer flags 0x%" PRIx64 + "...resetting session", + peer, peer->peer_gr_new_status_flag, + peer->flags); - switch (global_new_state) { - case GLOBAL_HELPER: - BGP_PEER_GR_HELPER_ENABLE(peer); - break; - case GLOBAL_GR: - BGP_PEER_GR_ENABLE(peer); - break; - case GLOBAL_DISABLE: - BGP_PEER_GR_DISABLE(peer); - break; - case GLOBAL_INVALID: - zlog_debug("%s [BGP_GR] GLOBAL_INVALID", - __func__); - return BGP_ERR_GR_OPERATION_FAILED; - } - } + /* Reset session to match with behavior for other peer + * configs that require the session to be re-setup. + */ + if (BGP_IS_VALID_STATE_FOR_NOTIF(peer->connection->status)) { + peer->last_reset = PEER_DOWN_CAPABILITY_CHANGE; + bgp_notify_send(peer->connection, BGP_NOTIFY_CEASE, + BGP_NOTIFY_CEASE_CONFIG_CHANGE); + } else + bgp_session_reset(peer); } - - bgp->global_gr_present_state = global_new_state; - - return BGP_GR_SUCCESS; } int bgp_gr_update_all(struct bgp *bgp, enum global_gr_command global_gr_cmd) @@ -2749,46 +2756,27 @@ int bgp_gr_update_all(struct bgp *bgp, enum global_gr_command global_gr_cmd) enum global_mode global_new_state = GLOBAL_INVALID; enum global_mode global_old_state = GLOBAL_INVALID; - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("%s [BGP_GR]START: global_gr_cmd :%s:", __func__, - print_global_gr_cmd(global_gr_cmd)); - global_old_state = bgp_global_gr_mode_get(bgp); + global_new_state = bgp->GLOBAL_GR_FSM[global_old_state][global_gr_cmd]; if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] global_old_gr_state :%s:", - print_global_gr_mode(global_old_state)); - - if (global_old_state != GLOBAL_INVALID) { - global_new_state = - bgp->GLOBAL_GR_FSM[global_old_state][global_gr_cmd]; + zlog_debug("%s: Handle GR command %s, current GR state %s, new GR state %s", + bgp->name_pretty, print_global_gr_cmd(global_gr_cmd), + print_global_gr_mode(global_old_state), + print_global_gr_mode(global_new_state)); - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] global_new_gr_state :%s:", - print_global_gr_mode(global_new_state)); - } else { - zlog_err("%s [BGP_GR] global_old_state == GLOBAL_INVALID", - __func__); + if (global_old_state == GLOBAL_INVALID) return BGP_ERR_GR_OPERATION_FAILED; - } - - if (global_new_state == GLOBAL_INVALID) { - zlog_err("%s [BGP_GR] global_new_state == GLOBAL_INVALID", - __func__); + if (global_new_state == GLOBAL_INVALID) return BGP_ERR_GR_INVALID_CMD; - } - if (global_new_state == global_old_state) { - /* Trace msg */ - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "%s [BGP_GR] global_new_state == global_old_state :%s", - __func__, - print_global_gr_mode(global_new_state)); + if (global_new_state == global_old_state) return BGP_GR_NO_OPERATION; - } - return bgp_gr_lookup_n_update_all_peer(bgp, global_new_state, - global_old_state); + /* Update global GR mode and process all peers in instance. */ + bgp->global_gr_present_state = global_new_state; + bgp_gr_update_mode_of_all_peers(bgp, global_new_state); + + return BGP_GR_SUCCESS; } const char *print_peer_gr_mode(enum peer_mode pr_mode) @@ -2903,179 +2891,101 @@ int bgp_neighbor_graceful_restart(struct peer *peer, { enum peer_mode peer_new_state = PEER_INVALID; enum peer_mode peer_old_state = PEER_INVALID; - struct bgp_peer_gr peer_state; + struct bgp_peer_gr gr_fsm; int result = BGP_GR_FAILURE; - /* - * fetch peer_old_state from peer structure also - * fetch global_old_state from bgp structure, - * peer had a back pointer to bgpo struct ; - */ - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("%s [BGP_GR] START:Peer: (%s) : peer_gr_cmd :%s:", - __func__, peer->host, - print_peer_gr_cmd(peer_gr_cmd)); - peer_old_state = bgp_peer_gr_mode_get(peer); + gr_fsm = peer->PEER_GR_FSM[peer_old_state][peer_gr_cmd]; + peer_new_state = gr_fsm.next_state; if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("%s [BGP_GR] peer_old_state: %d", __func__, - peer_old_state); + zlog_debug("%pBP: Handle GR command %s, current GR state %s, new GR state %s", + peer, print_peer_gr_cmd(peer_gr_cmd), + print_peer_gr_mode(peer_old_state), + print_peer_gr_mode(peer_new_state)); if (peer_old_state == PEER_INVALID) return BGP_ERR_GR_OPERATION_FAILED; - peer_state = peer->PEER_GR_FSM[peer_old_state][peer_gr_cmd]; - peer_new_state = peer_state.next_state; - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("%s [BGP_GR] peer_new_state: %d", __func__, - peer_new_state); - if (peer_new_state == PEER_INVALID) return BGP_ERR_GR_INVALID_CMD; - if (peer_new_state != peer_old_state) { - result = peer_state.action_fun(peer, peer_old_state, - peer_new_state); - } else { - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] peer_old_state == peer_new_state !"); + if (peer_new_state == peer_old_state) return BGP_GR_NO_OPERATION; - } - if (result == BGP_GR_SUCCESS) { - - /* Update the mode i.e peer_new_state into the peer structure */ - peer->peer_gr_present_state = peer_new_state; - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] Successfully change the state of the peer to : %s : !", - print_peer_gr_mode(peer_new_state)); - - return BGP_GR_SUCCESS; - } + result = gr_fsm.action_fun(peer, peer_old_state, peer_new_state); return result; } -unsigned int bgp_peer_gr_action(struct peer *peer, enum peer_mode old_peer_state, - enum peer_mode new_peer_state) +static inline bool gr_mode_matches(enum peer_mode peer_gr_mode, + enum global_mode global_gr_mode) { - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "%s [BGP_GR] Move peer from old_peer_state :%s: to new_peer_state :%s: !!!!", - __func__, print_peer_gr_mode(old_peer_state), - print_peer_gr_mode(new_peer_state)); + if ((peer_gr_mode == PEER_HELPER && global_gr_mode == GLOBAL_HELPER) || + (peer_gr_mode == PEER_GR && global_gr_mode == GLOBAL_GR) || + (peer_gr_mode == PEER_DISABLE && global_gr_mode == GLOBAL_DISABLE)) + return true; + return false; +} - enum global_mode bgp_gr_global_mode = GLOBAL_INVALID; - unsigned int ret = BGP_GR_FAILURE; +unsigned int bgp_peer_gr_action(struct peer *peer, enum peer_mode old_state, + enum peer_mode new_state) +{ + enum global_mode global_gr_mode = bgp_global_gr_mode_get(peer->bgp); + bool session_reset = true; - if (old_peer_state == new_peer_state) { - /* Nothing to do over here as the present and old state is the - * same */ + if (old_state == new_state) return BGP_GR_NO_OPERATION; - } - if ((old_peer_state == PEER_INVALID) - || (new_peer_state == PEER_INVALID)) { - /* something bad happend , print error message */ + if ((old_state == PEER_INVALID) || (new_state == PEER_INVALID)) return BGP_ERR_GR_INVALID_CMD; - } - - bgp_gr_global_mode = bgp_global_gr_mode_get(peer->bgp); - - if ((old_peer_state == PEER_GLOBAL_INHERIT) - && (new_peer_state != PEER_GLOBAL_INHERIT)) { - - /* fetch the Mode running in the Global state machine - *from the bgp structure into a variable called - *bgp_gr_global_mode - */ - /* Here we are checking if the - *1. peer_new_state == global_mode == helper_mode - *2. peer_new_state == global_mode == GR_mode - *3. peer_new_state == global_mode == disabled_mode - */ + global_gr_mode = bgp_global_gr_mode_get(peer->bgp); + if ((old_state == PEER_GLOBAL_INHERIT) && + (new_state != PEER_GLOBAL_INHERIT)) { BGP_PEER_GR_GLOBAL_INHERIT_UNSET(peer); - if ((int)new_peer_state == (int)bgp_gr_global_mode) { - /* This is incremental updates i.e no tear down - * of the existing session - * as the peer is already working in the same mode. + if (gr_mode_matches(new_state, global_gr_mode)) + /* Peer was inheriting the global state and + * its new state still is the same, so a + * session reset is not needed. */ - ret = BGP_GR_SUCCESS; - } else { - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] Peer state changed from :%s ", - print_peer_gr_mode(old_peer_state)); - - bgp_peer_move_to_gr_mode(peer, new_peer_state); - - ret = BGP_GR_SUCCESS; - } - } - /* In the case below peer is going into Global inherit mode i.e. - * the peer would work as the mode configured at the global level - */ - else if ((new_peer_state == PEER_GLOBAL_INHERIT) - && (old_peer_state != PEER_GLOBAL_INHERIT)) { - /* Here in this case it would be destructive - * in all the cases except one case when, - * Global GR is configured Disabled - * and present_peer_state is not disable - */ - + session_reset = false; + } else if ((new_state == PEER_GLOBAL_INHERIT) && + (old_state != PEER_GLOBAL_INHERIT)) { BGP_PEER_GR_GLOBAL_INHERIT_SET(peer); - if ((int)old_peer_state == (int)bgp_gr_global_mode) { - /* This is incremental updates - *i.e no tear down of the existing session - *as the peer is already working in the same mode. + if (gr_mode_matches(old_state, global_gr_mode)) + /* Peer is inheriting the global state and + * its old state was also the same, so a + * session reset is not needed. */ - ret = BGP_GR_SUCCESS; - } else { - /* Destructive always */ - /* Tear down the old session - * and send the new capability - * as per the bgp_gr_global_mode - */ - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] Peer state changed from :%s", - print_peer_gr_mode(old_peer_state)); + session_reset = false; + } - bgp_peer_move_to_gr_mode(peer, bgp_gr_global_mode); + /* Ensure we move to the new state and update flags */ + bgp_peer_move_to_gr_mode(peer, new_state); - ret = BGP_GR_SUCCESS; - } - } else { - /* - *This else case, it include all the cases except --> - *(new_peer_state != Peer_Global) && - *( old_peer_state != Peer_Global ) + if (session_reset) { + /* Reset session to match with behavior for other peer + * configs that require the session to be re-setup. */ - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] Peer state changed from :%s", - print_peer_gr_mode(old_peer_state)); - - bgp_peer_move_to_gr_mode(peer, new_peer_state); - - ret = BGP_GR_SUCCESS; + if (BGP_IS_VALID_STATE_FOR_NOTIF(peer->connection->status)) { + peer->last_reset = PEER_DOWN_CAPABILITY_CHANGE; + bgp_notify_send(peer->connection, BGP_NOTIFY_CEASE, + BGP_NOTIFY_CEASE_CONFIG_CHANGE); + } else + bgp_session_reset(peer); } - return ret; + return BGP_GR_SUCCESS; } -inline void bgp_peer_move_to_gr_mode(struct peer *peer, int new_state) +void bgp_peer_move_to_gr_mode(struct peer *peer, enum peer_mode new_state) { - int bgp_global_gr_mode = bgp_global_gr_mode_get(peer->bgp); + enum global_mode global_gr_mode = bgp_global_gr_mode_get(peer->bgp); + enum peer_mode old_state = bgp_peer_gr_mode_get(peer); switch (new_state) { case PEER_HELPER: @@ -3089,57 +2999,38 @@ inline void bgp_peer_move_to_gr_mode(struct peer *peer, int new_state) break; case PEER_GLOBAL_INHERIT: BGP_PEER_GR_GLOBAL_INHERIT_SET(peer); - - if (bgp_global_gr_mode == GLOBAL_HELPER) { - BGP_PEER_GR_HELPER_ENABLE(peer); - } else if (bgp_global_gr_mode == GLOBAL_GR) { - BGP_PEER_GR_ENABLE(peer); - } else if (bgp_global_gr_mode == GLOBAL_DISABLE) { - BGP_PEER_GR_DISABLE(peer); - } else { - zlog_err( - "[BGP_GR] Default switch inherit mode ::: SOMETHING IS WRONG !!!"); - } + bgp_peer_inherit_global_gr_mode(peer, global_gr_mode); break; + case PEER_INVALID: default: zlog_err( "[BGP_GR] Default switch mode ::: SOMETHING IS WRONG !!!"); break; } + bgp_peer_gr_flags_update(peer); + peer->peer_gr_present_state = new_state; + if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] Peer state changed --to--> : %d : !", - new_state); + zlog_debug("%pBP: Peer GR mode changed from %s to %s, GR flags 0x%x peer flags 0x%" PRIx64, + peer, print_peer_gr_mode(old_state), + print_peer_gr_mode(new_state), + peer->peer_gr_new_status_flag, peer->flags); } void bgp_peer_gr_flags_update(struct peer *peer) { - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("%s [BGP_GR] called !", __func__); if (CHECK_FLAG(peer->peer_gr_new_status_flag, PEER_GRACEFUL_RESTART_NEW_STATE_HELPER)) SET_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART_HELPER); else UNSET_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART_HELPER); - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] Peer %s Flag PEER_FLAG_GRACEFUL_RESTART_HELPER : %s : !", - peer->host, - (CHECK_FLAG(peer->flags, - PEER_FLAG_GRACEFUL_RESTART_HELPER) - ? "Set" - : "UnSet")); + if (CHECK_FLAG(peer->peer_gr_new_status_flag, PEER_GRACEFUL_RESTART_NEW_STATE_RESTART)) SET_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART); else UNSET_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART); - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] Peer %s Flag PEER_FLAG_GRACEFUL_RESTART : %s : !", - peer->host, - (CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART) - ? "Set" - : "UnSet")); + if (CHECK_FLAG(peer->peer_gr_new_status_flag, PEER_GRACEFUL_RESTART_NEW_STATE_INHERIT)) SET_FLAG(peer->flags, @@ -3147,28 +3038,28 @@ void bgp_peer_gr_flags_update(struct peer *peer) else UNSET_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART_GLOBAL_INHERIT); + if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] Peer %s Flag PEER_FLAG_GRACEFUL_RESTART_GLOBAL_INHERIT : %s : !", - peer->host, - (CHECK_FLAG(peer->flags, - PEER_FLAG_GRACEFUL_RESTART_GLOBAL_INHERIT) - ? "Set" - : "UnSet")); - - if (!CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART) - && !CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART_HELPER)) { - zlog_debug("[BGP_GR] Peer %s UNSET PEER_STATUS_NSF_MODE!", - peer->host); + zlog_debug("%pBP: Peer flags updated to 0x%" PRIx64 + ", GR flags 0x%x, GR mode %s", + peer, peer->flags, peer->peer_gr_new_status_flag, + print_peer_gr_mode(bgp_peer_gr_mode_get(peer))); + /* + * If GR has been completely disabled for the peer and we were + * acting as the Helper for the peer (i.e., keeping stale routes + * and running the restart timer or stalepath timer), clear those + * states. + */ + if (!CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART) && + !CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART_HELPER)) { UNSET_FLAG(peer->sflags, PEER_STATUS_NSF_MODE); if (CHECK_FLAG(peer->sflags, PEER_STATUS_NSF_WAIT)) { - + if (bgp_debug_neighbor_events(peer)) + zlog_debug("%pBP: GR disabled, stopping NSF and clearing stale routes", + peer); peer_nsf_stop(peer); - zlog_debug( - "[BGP_GR] Peer %s UNSET PEER_STATUS_NSF_WAIT!", - peer->host); } } } diff --git a/bgpd/bgp_fsm.h b/bgpd/bgp_fsm.h index bcdd49193fc3..85c488962fc9 100644 --- a/bgpd/bgp_fsm.h +++ b/bgpd/bgp_fsm.h @@ -151,7 +151,7 @@ int bgp_neighbor_graceful_restart(struct peer *peer, enum peer_gr_command peer_gr_cmd); unsigned int bgp_peer_gr_action(struct peer *peer, enum peer_mode old_peer_state, enum peer_mode new_peer_state); -void bgp_peer_move_to_gr_mode(struct peer *peer, int new_state); +void bgp_peer_move_to_gr_mode(struct peer *peer, enum peer_mode new_state); unsigned int bgp_peer_gr_helper_enable(struct peer *peer); unsigned int bgp_peer_gr_enable(struct peer *peer); unsigned int bgp_peer_gr_global_inherit(struct peer *peer); @@ -160,9 +160,6 @@ enum peer_mode bgp_peer_gr_mode_get(struct peer *peer); enum global_mode bgp_global_gr_mode_get(struct bgp *bgp); enum peer_mode bgp_get_peer_gr_mode_from_flags(struct peer *peer); unsigned int bgp_peer_gr_global_inherit_unset(struct peer *peer); -int bgp_gr_lookup_n_update_all_peer(struct bgp *bgp, - enum global_mode global_new_state, - enum global_mode global_old_state); void bgp_peer_gr_flags_update(struct peer *peer); const char *print_peer_gr_mode(enum peer_mode pr_mode); const char *print_peer_gr_cmd(enum peer_gr_command pr_gr_cmd); diff --git a/bgpd/bgp_main.c b/bgpd/bgp_main.c index 97658d340bf3..5e6a62c9b929 100644 --- a/bgpd/bgp_main.c +++ b/bgpd/bgp_main.c @@ -511,6 +511,7 @@ int main(int argc, char **argv) /* BGP master init. */ bgp_master_init(frr_init(), buffer_size, addresses); + bm->startup_time = monotime(NULL); bm->port = bgp_port; if (bgp_port == 0) bgp_option_set(BGP_OPT_NO_LISTEN); @@ -518,6 +519,9 @@ int main(int argc, char **argv) bgp_option_set(BGP_OPT_NO_FIB); if (no_zebra_flag) bgp_option_set(BGP_OPT_NO_ZEBRA); + if (bgpd_di.graceful_restart) + SET_FLAG(bm->flags, BM_FLAG_GRACEFUL_RESTART); + bgp_error_init(); /* Initializations. */ libagentx_init(); diff --git a/bgpd/bgp_open.c b/bgpd/bgp_open.c index 248d478fe1a3..945076709c5c 100644 --- a/bgpd/bgp_open.c +++ b/bgpd/bgp_open.c @@ -519,20 +519,17 @@ static int bgp_capability_restart(struct peer *peer, UNSET_FLAG(restart_flag_time, 0xF000); peer->v_gr_restart = restart_flag_time; - if (bgp_debug_neighbor_events(peer)) { - zlog_debug( - "%s Peer has%srestarted. Restart Time: %d, N-bit set: %s", - peer->host, - CHECK_FLAG(peer->cap, - PEER_CAP_GRACEFUL_RESTART_R_BIT_RCV) - ? " " - : " not ", - peer->v_gr_restart, - CHECK_FLAG(peer->cap, - PEER_CAP_GRACEFUL_RESTART_N_BIT_RCV) - ? "yes" - : "no"); - } + if (bgp_debug_neighbor_events(peer)) + zlog_debug("%pBP OPEN has GR capability, Restart time %d R-bit %s N-bit %s", + peer, peer->v_gr_restart, + CHECK_FLAG(peer->cap, + PEER_CAP_GRACEFUL_RESTART_R_BIT_RCV) + ? "SET" + : "NOT-SET", + CHECK_FLAG(peer->cap, + PEER_CAP_GRACEFUL_RESTART_N_BIT_RCV) + ? "SET" + : "NOT-SET"); while (stream_get_getp(s) + 4 <= end) { afi_t afi; @@ -556,14 +553,12 @@ static int bgp_capability_restart(struct peer *peer, iana_safi2str(pkt_safi)); } else { if (bgp_debug_neighbor_events(peer)) - zlog_debug( - "%s Address family %s is%spreserved", - peer->host, get_afi_safi_str(afi, safi, false), - CHECK_FLAG( - peer->af_cap[afi][safi], - PEER_CAP_RESTART_AF_PRESERVE_RCV) - ? " " - : " not "); + zlog_debug("%pBP F-bit %s for %s", peer, + CHECK_FLAG(peer->af_cap[afi][safi], + PEER_CAP_RESTART_AF_PRESERVE_RCV) + ? "SET" + : "NOT-SET", + get_afi_safi_str(afi, safi, false)); SET_FLAG(peer->af_cap[afi][safi], PEER_CAP_RESTART_AF_RCV); @@ -1587,15 +1582,12 @@ static void bgp_peer_send_gr_capability(struct stream *s, struct peer *peer, uint32_t restart_time; unsigned long capp = 0; unsigned long rcapp = 0; + struct bgp *bgp = peer->bgp; if (!CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART) && !CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART_HELPER)) return; - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] Sending helper Capability for Peer :%s :", - peer->host); - SET_FLAG(peer->cap, PEER_CAP_RESTART_ADV); stream_putc(s, BGP_OPEN_OPT_CAP); capp = stream_get_endp(s); /* Set Capability Len Pointer */ @@ -1605,42 +1597,41 @@ static void bgp_peer_send_gr_capability(struct stream *s, struct peer *peer, /* Set Restart Capability Len Pointer */ rcapp = stream_get_endp(s); stream_putc(s, 0); - restart_time = peer->bgp->restart_time; - if (peer->bgp->t_startup) { + restart_time = bgp->restart_time; + if (peer->bgp->t_startup || bgp_in_graceful_restart()) { SET_FLAG(restart_time, GRACEFUL_RESTART_R_BIT); SET_FLAG(peer->cap, PEER_CAP_GRACEFUL_RESTART_R_BIT_ADV); - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] Sending R-Bit for peer: %s", - peer->host); } - if (CHECK_FLAG(peer->bgp->flags, BGP_FLAG_GRACEFUL_NOTIFICATION)) { + if (CHECK_FLAG(bgp->flags, BGP_FLAG_GRACEFUL_NOTIFICATION)) { SET_FLAG(restart_time, GRACEFUL_RESTART_N_BIT); SET_FLAG(peer->cap, PEER_CAP_GRACEFUL_RESTART_N_BIT_ADV); - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] Sending N-Bit for peer: %s", - peer->host); } stream_putw(s, restart_time); + if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) + zlog_debug("%s: Sending GR Capability, Restart time %d R-bit %s, N-bit %s", + peer->host, bgp->restart_time, + CHECK_FLAG(peer->cap, + PEER_CAP_GRACEFUL_RESTART_R_BIT_ADV) + ? "SET" + : "NOT-SET", + CHECK_FLAG(peer->cap, + PEER_CAP_GRACEFUL_RESTART_N_BIT_ADV) + ? "SET" + : "NOT-SET"); + /* Send address-family specific graceful-restart capability * only when GR config is present */ if (CHECK_FLAG(peer->flags, PEER_FLAG_GRACEFUL_RESTART)) { - if (CHECK_FLAG(peer->bgp->flags, BGP_FLAG_GR_PRESERVE_FWD) - && BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] F bit Set"); - FOREACH_AFI_SAFI (afi, safi) { + bool f_bit = false; + if (!peer->afc[afi][safi]) continue; - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] Sending GR Capability for AFI :%d :, SAFI :%d:", - afi, safi); - /* Convert AFI, SAFI to values for * packet. */ @@ -1648,11 +1639,15 @@ static void bgp_peer_send_gr_capability(struct stream *s, struct peer *peer, &pkt_safi); stream_putw(s, pkt_afi); stream_putc(s, pkt_safi); - if (CHECK_FLAG(peer->bgp->flags, - BGP_FLAG_GR_PRESERVE_FWD)) - stream_putc(s, GRACEFUL_RESTART_F_BIT); - else - stream_putc(s, 0); + + f_bit = bgp_gr_is_forwarding_preserved(bgp); + + if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) + zlog_debug("... F-bit %s for %s", + f_bit ? "SET" : "NOT-SET", + get_afi_safi_str(afi, safi, false)); + + stream_putc(s, f_bit ? GRACEFUL_RESTART_F_BIT : 0); } } diff --git a/bgpd/bgp_packet.c b/bgpd/bgp_packet.c index 8825136e1768..9a047b9d456f 100644 --- a/bgpd/bgp_packet.c +++ b/bgpd/bgp_packet.c @@ -696,10 +696,9 @@ void bgp_open_send(struct peer_connection *connection) bgp_packet_set_size(s); if (bgp_debug_neighbor_events(peer)) - zlog_debug( - "%s sending OPEN, version %d, my as %u, holdtime %d, id %pI4", - peer->host, BGP_VERSION_4, local_as, send_holdtime, - &peer->local_id); + zlog_debug("%pBP fd %d sending OPEN, version %d, my as %u, holdtime %d, id %pI4", + peer, peer->connection->fd, BGP_VERSION_4, local_as, + send_holdtime, &peer->local_id); /* Dump packet if debug option is set. */ /* bgp_packet_dump (s); */ @@ -1296,7 +1295,7 @@ void bgp_capability_send(struct peer *peer, afi_t afi, safi_t safi, stream_putc(s, 0); gr_restart_time = peer->bgp->restart_time; - if (peer->bgp->t_startup) { + if (peer->bgp->t_startup || bgp_in_graceful_restart()) { SET_FLAG(gr_restart_time, GRACEFUL_RESTART_R_BIT); SET_FLAG(peer->cap, PEER_CAP_GRACEFUL_RESTART_R_BIT_ADV); } diff --git a/bgpd/bgp_route.c b/bgpd/bgp_route.c index 9c3602311f77..c7e7b7a740ac 100644 --- a/bgpd/bgp_route.c +++ b/bgpd/bgp_route.c @@ -3877,6 +3877,7 @@ void bgp_best_path_select_defer(struct bgp *bgp, afi_t afi, safi_t safi) struct bgp_dest *dest; int cnt = 0; struct afi_safi_info *thread_info; + bool route_sync_pending = false; if (bgp->gr_info[afi][safi].t_route_select) { struct event *t = bgp->gr_info[afi][safi].t_route_select; @@ -3886,7 +3887,7 @@ void bgp_best_path_select_defer(struct bgp *bgp, afi_t afi, safi_t safi) EVENT_OFF(bgp->gr_info[afi][safi].t_route_select); } - if (BGP_DEBUG(update, UPDATE_OUT)) { + if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) { zlog_debug("%s: processing route for %s : cnt %d", __func__, get_afi_safi_str(afi, safi, false), bgp->gr_info[afi][safi].gr_deferred); @@ -3919,6 +3920,21 @@ void bgp_best_path_select_defer(struct bgp *bgp, afi_t afi, safi_t safi) /* Send route processing complete message to RIB */ bgp_zebra_update(bgp, afi, safi, ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE); + bgp->gr_info[afi][safi].route_sync = true; + + /* If this instance is all done, check for GR completion overall */ + FOREACH_AFI_SAFI_NSF (afi, safi) { + if (bgp->gr_info[afi][safi].af_enabled && + !bgp->gr_info[afi][safi].route_sync) { + route_sync_pending = true; + break; + } + } + + if (!route_sync_pending) { + bgp->gr_route_sync_pending = false; + bgp_update_gr_completion(); + } return; } diff --git a/bgpd/bgp_vty.c b/bgpd/bgp_vty.c index 230fedf4ece8..e9a79766ec84 100644 --- a/bgpd/bgp_vty.c +++ b/bgpd/bgp_vty.c @@ -3019,6 +3019,98 @@ DEFUN (no_bgp_deterministic_med, return CMD_SUCCESS; } +static int bgp_inst_gr_config_vty(struct vty *vty, struct bgp *bgp, bool on, + bool disable) +{ + int ret = BGP_GR_FAILURE; + + /* + * Update the instance and all its peers, if appropriate. + * Then, inform zebra of BGP's GR capabilities, if needed. + */ + if (disable) + ret = bgp_gr_update_all(bgp, on ? GLOBAL_DISABLE_CMD + : NO_GLOBAL_DISABLE_CMD); + else + ret = bgp_gr_update_all(bgp, + on ? GLOBAL_GR_CMD : NO_GLOBAL_GR_CMD); + + VTY_BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(bgp, bgp->peer, + ret); + return ret; +} + +static int bgp_global_gr_config_vty(struct vty *vty, bool on, bool disable) +{ + struct listnode *node, *nnode; + struct bgp *bgp; + bool vrf_cfg = false; + int ret = BGP_GR_FAILURE; + + if (disable) { + if ((on && CHECK_FLAG(bm->flags, BM_FLAG_GR_DISABLED)) || + (!on && !CHECK_FLAG(bm->flags, BM_FLAG_GR_DISABLED))) + return CMD_SUCCESS; + } else { + if ((on && CHECK_FLAG(bm->flags, BM_FLAG_GR_RESTARTER)) || + (!on && !CHECK_FLAG(bm->flags, BM_FLAG_GR_RESTARTER))) + return CMD_SUCCESS; + } + + /* See if GR is set per-vrf and warn user to delete */ + if (!CHECK_FLAG(bm->flags, BM_FLAG_GR_CONFIGURED)) { + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) { + enum global_mode gr_mode = bgp_global_gr_mode_get(bgp); + + if (gr_mode != GLOBAL_HELPER) { + vty_out(vty, + "%% graceful-restart configuration found in %s, mode %d\n", + bgp->name_pretty, gr_mode); + vrf_cfg = true; + } + } + } + + if (vrf_cfg) { + vty_out(vty, + "%%Failed: global graceful-restart not permitted with per-vrf configuration\n"); + return CMD_WARNING; + } + + /* Set flag globally */ + if (on) { + if (disable) { + UNSET_FLAG(bm->flags, BM_FLAG_GR_RESTARTER); + SET_FLAG(bm->flags, BM_FLAG_GR_DISABLED); + } else { + SET_FLAG(bm->flags, BM_FLAG_GR_RESTARTER); + UNSET_FLAG(bm->flags, BM_FLAG_GR_DISABLED); + } + } else { + if (disable) + UNSET_FLAG(bm->flags, BM_FLAG_GR_DISABLED); + else + UNSET_FLAG(bm->flags, BM_FLAG_GR_RESTARTER); + } + + /* Initiate processing for all BGP instances. */ + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) { + ret = bgp_inst_gr_config_vty(vty, bgp, on, disable); + if (ret != BGP_GR_SUCCESS) + vty_out(vty, + "%% Applying global graceful-restart %s config to vrf %s failed, error %d\n", + (disable) ? "disable" : "", + bgp->inst_type == BGP_INSTANCE_TYPE_DEFAULT + ? "Default" + : bgp->name, + ret); + } + + vty_out(vty, + "Graceful restart configuration changed, reset all peers to take effect\n"); + return bgp_vty_return(vty, ret); +} + /* "bgp graceful-restart mode" configuration. */ DEFUN (bgp_graceful_restart, bgp_graceful_restart_cmd, @@ -3027,25 +3119,18 @@ DEFUN (bgp_graceful_restart, GR_CMD ) { - int ret = BGP_GR_FAILURE; - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] bgp_graceful_restart_cmd : START "); + if (vty->node == CONFIG_NODE) + return bgp_global_gr_config_vty(vty, true, false); + int ret = BGP_GR_FAILURE; VTY_DECLVAR_CONTEXT(bgp, bgp); - ret = bgp_gr_update_all(bgp, GLOBAL_GR_CMD); + ret = bgp_inst_gr_config_vty(vty, bgp, true, false); if (ret == BGP_GR_SUCCESS) { - VTY_BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(bgp, - bgp->peer, - ret); vty_out(vty, "Graceful restart configuration changed, reset all peers to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] bgp_graceful_restart_cmd : END "); - return bgp_vty_return(vty, ret); } @@ -3057,14 +3142,13 @@ DEFUN (no_bgp_graceful_restart, NO_GR_CMD ) { - VTY_DECLVAR_CONTEXT(bgp, bgp); - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] no_bgp_graceful_restart_cmd : START "); + if (vty->node == CONFIG_NODE) + return bgp_global_gr_config_vty(vty, false, false); + VTY_DECLVAR_CONTEXT(bgp, bgp); int ret = BGP_GR_FAILURE; - ret = bgp_gr_update_all(bgp, NO_GLOBAL_GR_CMD); + ret = bgp_inst_gr_config_vty(vty, bgp, false, false); if (ret == BGP_GR_SUCCESS) { VTY_BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(bgp, bgp->peer, @@ -3073,9 +3157,6 @@ DEFUN (no_bgp_graceful_restart, "Graceful restart configuration changed, reset all peers to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] no_bgp_graceful_restart_cmd : END "); - return bgp_vty_return(vty, ret); } @@ -3087,12 +3168,21 @@ DEFUN (bgp_graceful_restart_stalepath_time, "Set the max time to hold onto restarting peer's stale paths\n" "Delay value (seconds)\n") { - VTY_DECLVAR_CONTEXT(bgp, bgp); int idx_number = 3; uint32_t stalepath; stalepath = strtoul(argv[idx_number]->arg, NULL, 10); - bgp->stalepath_time = stalepath; + if (vty->node == CONFIG_NODE) { + struct listnode *node, *nnode; + struct bgp *bgp; + + bm->stalepath_time = stalepath; + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) + bgp->stalepath_time = stalepath; + } else { + VTY_DECLVAR_CONTEXT(bgp, bgp); + bgp->stalepath_time = stalepath; + } return CMD_SUCCESS; } @@ -3104,20 +3194,32 @@ DEFUN (bgp_graceful_restart_restart_time, "Set the time to wait to delete stale routes before a BGP open message is received\n" "Delay value (seconds)\n") { - VTY_DECLVAR_CONTEXT(bgp, bgp); int idx_number = 3; uint32_t restart; struct listnode *node, *nnode; struct peer *peer; restart = strtoul(argv[idx_number]->arg, NULL, 10); - bgp->restart_time = restart; - for (ALL_LIST_ELEMENTS(bgp->peer, node, nnode, peer)) - bgp_capability_send(peer, AFI_IP, SAFI_UNICAST, - CAPABILITY_CODE_RESTART, - CAPABILITY_ACTION_SET); + if (vty->node == CONFIG_NODE) { + struct bgp *bgp; + bm->restart_time = restart; + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) { + bgp->restart_time = restart; + for (ALL_LIST_ELEMENTS(bgp->peer, node, nnode, peer)) + bgp_capability_send(peer, AFI_IP, SAFI_UNICAST, + CAPABILITY_CODE_RESTART, + CAPABILITY_ACTION_SET); + } + } else { + VTY_DECLVAR_CONTEXT(bgp, bgp); + bgp->restart_time = restart; + for (ALL_LIST_ELEMENTS(bgp->peer, node, nnode, peer)) + bgp_capability_send(peer, AFI_IP, SAFI_UNICAST, + CAPABILITY_CODE_RESTART, + CAPABILITY_ACTION_SET); + } return CMD_SUCCESS; } @@ -3129,16 +3231,32 @@ DEFUN (bgp_graceful_restart_select_defer_time, "Set the time to defer the BGP route selection after restart\n" "Delay value (seconds, 0 - disable)\n") { - VTY_DECLVAR_CONTEXT(bgp, bgp); int idx_number = 3; uint32_t defer_time; defer_time = strtoul(argv[idx_number]->arg, NULL, 10); - bgp->select_defer_time = defer_time; - if (defer_time == 0) - SET_FLAG(bgp->flags, BGP_FLAG_SELECT_DEFER_DISABLE); - else - UNSET_FLAG(bgp->flags, BGP_FLAG_SELECT_DEFER_DISABLE); + if (vty->node == CONFIG_NODE) { + struct listnode *node, *nnode; + struct bgp *bgp; + + bm->select_defer_time = defer_time; + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) { + bgp->select_defer_time = defer_time; + if (defer_time == 0) + SET_FLAG(bgp->flags, + BGP_FLAG_SELECT_DEFER_DISABLE); + else + UNSET_FLAG(bgp->flags, + BGP_FLAG_SELECT_DEFER_DISABLE); + } + } else { + VTY_DECLVAR_CONTEXT(bgp, bgp); + bgp->select_defer_time = defer_time; + if (defer_time == 0) + SET_FLAG(bgp->flags, BGP_FLAG_SELECT_DEFER_DISABLE); + else + UNSET_FLAG(bgp->flags, BGP_FLAG_SELECT_DEFER_DISABLE); + } return CMD_SUCCESS; } @@ -3152,9 +3270,17 @@ DEFUN (no_bgp_graceful_restart_stalepath_time, "Set the max time to hold onto restarting peer's stale paths\n" "Delay value (seconds)\n") { - VTY_DECLVAR_CONTEXT(bgp, bgp); + if (vty->node == CONFIG_NODE) { + struct listnode *node, *nnode; + struct bgp *bgp; - bgp->stalepath_time = BGP_DEFAULT_STALEPATH_TIME; + bm->stalepath_time = BGP_DEFAULT_STALEPATH_TIME; + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) + bgp->stalepath_time = BGP_DEFAULT_STALEPATH_TIME; + } else { + VTY_DECLVAR_CONTEXT(bgp, bgp); + bgp->stalepath_time = BGP_DEFAULT_STALEPATH_TIME; + } return CMD_SUCCESS; } @@ -3167,17 +3293,30 @@ DEFUN (no_bgp_graceful_restart_restart_time, "Set the time to wait to delete stale routes before a BGP open message is received\n" "Delay value (seconds)\n") { - VTY_DECLVAR_CONTEXT(bgp, bgp); struct listnode *node, *nnode; struct peer *peer; - bgp->restart_time = BGP_DEFAULT_RESTART_TIME; + if (vty->node == CONFIG_NODE) { + struct bgp *bgp; - for (ALL_LIST_ELEMENTS(bgp->peer, node, nnode, peer)) - bgp_capability_send(peer, AFI_IP, SAFI_UNICAST, - CAPABILITY_CODE_RESTART, - CAPABILITY_ACTION_UNSET); + bm->restart_time = BGP_DEFAULT_RESTART_TIME; + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) { + bgp->restart_time = BGP_DEFAULT_RESTART_TIME; + + for (ALL_LIST_ELEMENTS(bgp->peer, node, nnode, peer)) + bgp_capability_send(peer, AFI_IP, SAFI_UNICAST, + CAPABILITY_CODE_RESTART, + CAPABILITY_ACTION_UNSET); + } + } else { + VTY_DECLVAR_CONTEXT(bgp, bgp); + bgp->restart_time = BGP_DEFAULT_RESTART_TIME; + for (ALL_LIST_ELEMENTS(bgp->peer, node, nnode, peer)) + bgp_capability_send(peer, AFI_IP, SAFI_UNICAST, + CAPABILITY_CODE_RESTART, + CAPABILITY_ACTION_UNSET); + } return CMD_SUCCESS; } @@ -3190,10 +3329,21 @@ DEFUN (no_bgp_graceful_restart_select_defer_time, "Set the time to defer the BGP route selection after restart\n" "Delay value (seconds)\n") { - VTY_DECLVAR_CONTEXT(bgp, bgp); + if (vty->node == CONFIG_NODE) { + struct listnode *node, *nnode; + struct bgp *bgp; - bgp->select_defer_time = BGP_DEFAULT_SELECT_DEFERRAL_TIME; - UNSET_FLAG(bgp->flags, BGP_FLAG_SELECT_DEFER_DISABLE); + bm->select_defer_time = BGP_DEFAULT_SELECT_DEFERRAL_TIME; + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) { + bgp->select_defer_time = + BGP_DEFAULT_SELECT_DEFERRAL_TIME; + UNSET_FLAG(bgp->flags, BGP_FLAG_SELECT_DEFER_DISABLE); + } + } else { + VTY_DECLVAR_CONTEXT(bgp, bgp); + bgp->select_defer_time = BGP_DEFAULT_SELECT_DEFERRAL_TIME; + UNSET_FLAG(bgp->flags, BGP_FLAG_SELECT_DEFER_DISABLE); + } return CMD_SUCCESS; } @@ -3205,8 +3355,17 @@ DEFUN (bgp_graceful_restart_preserve_fw, "Graceful restart capability parameters\n" "Sets F-bit indication that fib is preserved while doing Graceful Restart\n") { - VTY_DECLVAR_CONTEXT(bgp, bgp); - SET_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD); + if (vty->node == CONFIG_NODE) { + struct listnode *node, *nnode; + struct bgp *bgp; + + SET_FLAG(bm->flags, BM_FLAG_GR_PRESERVE_FWD); + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) + SET_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD); + } else { + VTY_DECLVAR_CONTEXT(bgp, bgp); + SET_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD); + } return CMD_SUCCESS; } @@ -3218,8 +3377,17 @@ DEFUN (no_bgp_graceful_restart_preserve_fw, "Graceful restart capability parameters\n" "Unsets F-bit indication that fib is preserved while doing Graceful Restart\n") { - VTY_DECLVAR_CONTEXT(bgp, bgp); - UNSET_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD); + if (vty->node == CONFIG_NODE) { + struct listnode *node, *nnode; + struct bgp *bgp; + + UNSET_FLAG(bm->flags, BM_FLAG_GR_PRESERVE_FWD); + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) + UNSET_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD); + } else { + VTY_DECLVAR_CONTEXT(bgp, bgp); + UNSET_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD); + } return CMD_SUCCESS; } @@ -3271,21 +3439,17 @@ DEFUN (bgp_graceful_restart_disable, BGP_STR GR_DISABLE) { + if (vty->node == CONFIG_NODE) + return bgp_global_gr_config_vty(vty, true, true); + int ret = BGP_GR_FAILURE; struct listnode *node, *nnode; struct peer *peer; - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] bgp_graceful_restart_disable_cmd : START "); - VTY_DECLVAR_CONTEXT(bgp, bgp); - ret = bgp_gr_update_all(bgp, GLOBAL_DISABLE_CMD); + ret = bgp_inst_gr_config_vty(vty, bgp, true, true); if (ret == BGP_GR_SUCCESS) { - VTY_BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(bgp, - bgp->peer, - ret); vty_out(vty, "Graceful restart configuration changed, reset all peers to take effect\n"); @@ -3299,9 +3463,6 @@ DEFUN (bgp_graceful_restart_disable, } } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug("[BGP_GR] bgp_graceful_restart_disable_cmd : END "); - return bgp_vty_return(vty, ret); } @@ -3313,27 +3474,18 @@ DEFUN (no_bgp_graceful_restart_disable, NO_GR_DISABLE ) { - VTY_DECLVAR_CONTEXT(bgp, bgp); - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] no_bgp_graceful_restart_disable_cmd : START "); + if (vty->node == CONFIG_NODE) + return bgp_global_gr_config_vty(vty, false, true); + VTY_DECLVAR_CONTEXT(bgp, bgp); int ret = BGP_GR_FAILURE; - ret = bgp_gr_update_all(bgp, NO_GLOBAL_DISABLE_CMD); + ret = bgp_inst_gr_config_vty(vty, bgp, false, true); if (ret == BGP_GR_SUCCESS) { - VTY_BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(bgp, - bgp->peer, - ret); vty_out(vty, "Graceful restart configuration changed, reset all peers to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] no_bgp_graceful_restart_disable_cmd : END "); - return bgp_vty_return(vty, ret); } @@ -3351,13 +3503,14 @@ DEFUN (bgp_neighbor_graceful_restart_set, VTY_BGP_GR_DEFINE_LOOP_VARIABLE; - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] bgp_neighbor_graceful_restart_set_cmd : START "); - peer = peer_and_group_lookup_vty(vty, argv[idx_peer]->arg); if (!peer) return CMD_WARNING_CONFIG_FAILED; + if (CHECK_FLAG(peer->sflags, PEER_STATUS_GROUP)) { + vty_out(vty, + "Per peer-group graceful-restart configuration is not yet supported\n"); + return CMD_WARNING_CONFIG_FAILED; + } result = bgp_neighbor_graceful_restart(peer, PEER_GR_CMD); if (result == BGP_GR_SUCCESS) { @@ -3367,15 +3520,7 @@ DEFUN (bgp_neighbor_graceful_restart_set, "Graceful restart configuration changed, reset this peer to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] bgp_neighbor_graceful_restart_set_cmd : END "); - - if (ret != BGP_GR_SUCCESS) - vty_out(vty, - "As part of configuring graceful-restart, capability send to zebra failed\n"); - - return bgp_vty_return(vty, result); + return bgp_vty_return(vty, ret); } DEFUN (no_bgp_neighbor_graceful_restart, @@ -3396,10 +3541,11 @@ DEFUN (no_bgp_neighbor_graceful_restart, peer = peer_and_group_lookup_vty(vty, argv[idx_peer]->arg); if (!peer) return CMD_WARNING_CONFIG_FAILED; - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] no_bgp_neighbor_graceful_restart_set_cmd : START "); + if (CHECK_FLAG(peer->sflags, PEER_STATUS_GROUP)) { + vty_out(vty, + "Per peer-group graceful-restart configuration is not yet supported\n"); + return CMD_WARNING_CONFIG_FAILED; + } result = bgp_neighbor_graceful_restart(peer, NO_PEER_GR_CMD); if (ret == BGP_GR_SUCCESS) { @@ -3409,14 +3555,6 @@ DEFUN (no_bgp_neighbor_graceful_restart, "Graceful restart configuration changed, reset this peer to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] no_bgp_neighbor_graceful_restart_set_cmd : END "); - - if (ret != BGP_GR_SUCCESS) - vty_out(vty, - "As part of configuring graceful-restart, capability send to zebra failed\n"); - return bgp_vty_return(vty, result); } @@ -3434,15 +3572,14 @@ DEFUN (bgp_neighbor_graceful_restart_helper_set, VTY_BGP_GR_DEFINE_LOOP_VARIABLE; - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] bgp_neighbor_graceful_restart_helper_set_cmd : START "); - peer = peer_and_group_lookup_vty(vty, argv[idx_peer]->arg); - if (!peer) return CMD_WARNING_CONFIG_FAILED; - + if (CHECK_FLAG(peer->sflags, PEER_STATUS_GROUP)) { + vty_out(vty, + "Per peer-group graceful-restart configuration is not yet supported\n"); + return CMD_WARNING_CONFIG_FAILED; + } ret = bgp_neighbor_graceful_restart(peer, PEER_HELPER_CMD); if (ret == BGP_GR_SUCCESS) { @@ -3452,10 +3589,6 @@ DEFUN (bgp_neighbor_graceful_restart_helper_set, "Graceful restart configuration changed, reset this peer to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] bgp_neighbor_graceful_restart_helper_set_cmd : END "); - return bgp_vty_return(vty, ret); } @@ -3477,10 +3610,11 @@ DEFUN (no_bgp_neighbor_graceful_restart_helper, peer = peer_and_group_lookup_vty(vty, argv[idx_peer]->arg); if (!peer) return CMD_WARNING_CONFIG_FAILED; - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] no_bgp_neighbor_graceful_restart_helper_set_cmd : START "); + if (CHECK_FLAG(peer->sflags, PEER_STATUS_GROUP)) { + vty_out(vty, + "Per peer-group graceful-restart configuration is not yet supported\n"); + return CMD_WARNING_CONFIG_FAILED; + } ret = bgp_neighbor_graceful_restart(peer, NO_PEER_HELPER_CMD); if (ret == BGP_GR_SUCCESS) { @@ -3490,10 +3624,6 @@ DEFUN (no_bgp_neighbor_graceful_restart_helper, "Graceful restart configuration changed, reset this peer to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] no_bgp_neighbor_graceful_restart_helper_set_cmd : END "); - return bgp_vty_return(vty, ret); } @@ -3511,29 +3641,24 @@ DEFUN (bgp_neighbor_graceful_restart_disable_set, VTY_BGP_GR_DEFINE_LOOP_VARIABLE; - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] bgp_neighbor_graceful_restart_disable_set_cmd : START "); - peer = peer_and_group_lookup_vty(vty, argv[idx_peer]->arg); if (!peer) return CMD_WARNING_CONFIG_FAILED; + if (CHECK_FLAG(peer->sflags, PEER_STATUS_GROUP)) { + vty_out(vty, + "Per peer-group graceful-restart configuration is not yet supported\n"); + return CMD_WARNING_CONFIG_FAILED; + } ret = bgp_neighbor_graceful_restart(peer, PEER_DISABLE_CMD); if (ret == BGP_GR_SUCCESS) { - if (peer->bgp->t_startup) + if (peer->bgp->t_startup || bgp_in_graceful_restart()) bgp_peer_gr_flags_update(peer); VTY_BGP_GR_ROUTER_DETECT(bgp, peer, peer->bgp->peer); VTY_SEND_BGP_GR_CAPABILITY_TO_ZEBRA(peer->bgp, ret); - vty_out(vty, - "Graceful restart configuration changed, reset this peer to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR]bgp_neighbor_graceful_restart_disable_set_cmd : END "); - return bgp_vty_return(vty, ret); } @@ -3555,23 +3680,18 @@ DEFUN (no_bgp_neighbor_graceful_restart_disable, peer = peer_and_group_lookup_vty(vty, argv[idx_peer]->arg); if (!peer) return CMD_WARNING_CONFIG_FAILED; - - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] no_bgp_neighbor_graceful_restart_disable_set_cmd : START "); + if (CHECK_FLAG(peer->sflags, PEER_STATUS_GROUP)) { + vty_out(vty, + "Per peer-group graceful-restart configuration is not yet supported\n"); + return CMD_WARNING_CONFIG_FAILED; + } ret = bgp_neighbor_graceful_restart(peer, NO_PEER_DISABLE_CMD); if (ret == BGP_GR_SUCCESS) { VTY_BGP_GR_ROUTER_DETECT(bgp, peer, peer->bgp->peer); VTY_SEND_BGP_GR_CAPABILITY_TO_ZEBRA(peer->bgp, ret); - vty_out(vty, - "Graceful restart configuration changed, reset this peer to take effect\n"); } - if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) - zlog_debug( - "[BGP_GR] no_bgp_neighbor_graceful_restart_disable_set_cmd : END "); - return bgp_vty_return(vty, ret); } @@ -3586,9 +3706,10 @@ DEFPY (neighbor_graceful_shutdown, afi_t afi; safi_t safi; struct peer *peer; - VTY_DECLVAR_CONTEXT(bgp, bgp); int ret; + VTY_DECLVAR_CONTEXT(bgp, bgp); + peer = peer_and_group_lookup_vty(vty, neighbor); if (!peer) return CMD_WARNING_CONFIG_FAILED; @@ -19166,6 +19287,7 @@ int bgp_config_write(struct vty *vty) hook_call(bgp_snmp_traps_config_write, vty); + vty_out(vty, "!\n"); if (bm->rmap_update_timer != RMAP_DEFAULT_UPDATE_TIMER) vty_out(vty, "bgp route-map delay-timer %u\n", bm->rmap_update_timer); @@ -19180,6 +19302,30 @@ int bgp_config_write(struct vty *vty) if (bm->wait_for_fib) vty_out(vty, "bgp suppress-fib-pending\n"); + if (bm->stalepath_time != BGP_DEFAULT_STALEPATH_TIME) + vty_out(vty, "bgp graceful-restart stalepath-time %u\n", + bm->stalepath_time); + + if (bm->restart_time != BGP_DEFAULT_RESTART_TIME) + vty_out(vty, "bgp graceful-restart restart-time %u\n", + bm->restart_time); + + if (bm->select_defer_time != BGP_DEFAULT_SELECT_DEFERRAL_TIME) + vty_out(vty, "bgp graceful-restart select-defer-time %u\n", + bm->select_defer_time); + + if (CHECK_FLAG(bm->flags, BM_FLAG_GR_RESTARTER)) + vty_out(vty, "bgp graceful-restart\n"); + else if (CHECK_FLAG(bm->flags, BM_FLAG_GR_DISABLED)) + vty_out(vty, "bgp graceful-restart-disable\n"); + + if (CHECK_FLAG(bm->flags, BM_FLAG_GR_PRESERVE_FWD)) + vty_out(vty, "bgp graceful-restart preserve-fw-state\n"); + + if (bm->rib_stale_time != BGP_DEFAULT_RIB_STALE_TIME) + vty_out(vty, "bgp graceful-restart rib-stale-time %u\n", + bm->rib_stale_time); + if (CHECK_FLAG(bm->flags, BM_FLAG_GRACEFUL_SHUTDOWN)) vty_out(vty, "bgp graceful-shutdown\n"); @@ -19201,6 +19347,8 @@ int bgp_config_write(struct vty *vty) if (bm->outq_limit != BM_DEFAULT_Q_LIMIT) vty_out(vty, "bgp output-queue-limit %u\n", bm->outq_limit); + vty_out(vty, "!\n"); + /* BGP configuration. */ for (ALL_LIST_ELEMENTS(bm->bgp, mnode, mnnode, bgp)) { @@ -19442,15 +19590,21 @@ int bgp_config_write(struct vty *vty) " bgp long-lived-graceful-restart stale-time %u\n", bgp->llgr_stale_time); - /* BGP graceful-restart. */ - if (bgp->stalepath_time != BGP_DEFAULT_STALEPATH_TIME) - vty_out(vty, - " bgp graceful-restart stalepath-time %u\n", - bgp->stalepath_time); + /* BGP per-instance graceful-restart. */ + /* BGP-wide settings and per-instance settings are mutually + * exclusive. + */ + if (bm->stalepath_time == BGP_DEFAULT_STALEPATH_TIME) + if (bgp->stalepath_time != BGP_DEFAULT_STALEPATH_TIME) + vty_out(vty, + " bgp graceful-restart stalepath-time %u\n", + bgp->stalepath_time); - if (bgp->restart_time != BGP_DEFAULT_RESTART_TIME) - vty_out(vty, " bgp graceful-restart restart-time %u\n", - bgp->restart_time); + if (bm->restart_time == BGP_DEFAULT_RESTART_TIME) + if (bgp->restart_time != BGP_DEFAULT_RESTART_TIME) + vty_out(vty, + " bgp graceful-restart restart-time %u\n", + bgp->restart_time); if (!!CHECK_FLAG(bgp->flags, BGP_FLAG_GRACEFUL_NOTIFICATION) != SAVE_BGP_GRACEFUL_NOTIFICATION) @@ -19460,30 +19614,34 @@ int bgp_config_write(struct vty *vty) ? "" : "no "); - if (bgp->select_defer_time != BGP_DEFAULT_SELECT_DEFERRAL_TIME) - vty_out(vty, - " bgp graceful-restart select-defer-time %u\n", - bgp->select_defer_time); + if (bm->select_defer_time == BGP_DEFAULT_SELECT_DEFERRAL_TIME) + if (bgp->select_defer_time != + BGP_DEFAULT_SELECT_DEFERRAL_TIME) + vty_out(vty, + " bgp graceful-restart select-defer-time %u\n", + bgp->select_defer_time); - if (bgp_global_gr_mode_get(bgp) == GLOBAL_GR) - vty_out(vty, " bgp graceful-restart\n"); + if (!CHECK_FLAG(bm->flags, BM_FLAG_GR_CONFIGURED)) { + if (bgp_global_gr_mode_get(bgp) == GLOBAL_GR) + vty_out(vty, " bgp graceful-restart\n"); - if (bgp_global_gr_mode_get(bgp) == GLOBAL_DISABLE) - vty_out(vty, " bgp graceful-restart-disable\n"); + if (bgp_global_gr_mode_get(bgp) == GLOBAL_DISABLE) + vty_out(vty, " bgp graceful-restart-disable\n"); + } - /* BGP graceful-restart Preserve State F bit. */ - if (CHECK_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD)) - vty_out(vty, - " bgp graceful-restart preserve-fw-state\n"); + if (!CHECK_FLAG(bm->flags, BM_FLAG_GR_PRESERVE_FWD)) + if (CHECK_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD)) + vty_out(vty, + " bgp graceful-restart preserve-fw-state\n"); /* BGP TCP keepalive */ bgp_config_tcp_keepalive(vty, bgp); - /* Stale timer for RIB */ - if (bgp->rib_stale_time != BGP_DEFAULT_RIB_STALE_TIME) - vty_out(vty, - " bgp graceful-restart rib-stale-time %u\n", - bgp->rib_stale_time); + if (bm->rib_stale_time == BGP_DEFAULT_RIB_STALE_TIME) + if (bgp->rib_stale_time != BGP_DEFAULT_RIB_STALE_TIME) + vty_out(vty, + " bgp graceful-restart rib-stale-time %u\n", + bgp->rib_stale_time); /* BGP bestpath method. */ if (CHECK_FLAG(bgp->flags, BGP_FLAG_ASPATH_IGNORE)) @@ -20142,6 +20300,26 @@ void bgp_vty_init(void) install_element(CONFIG_NODE, &bgp_graceful_shutdown_cmd); install_element(CONFIG_NODE, &no_bgp_graceful_shutdown_cmd); + /* BGP-wide graceful-restart commands. */ + install_element(CONFIG_NODE, &bgp_graceful_restart_cmd); + install_element(CONFIG_NODE, &no_bgp_graceful_restart_cmd); + install_element(CONFIG_NODE, &bgp_graceful_restart_disable_cmd); + install_element(CONFIG_NODE, &no_bgp_graceful_restart_disable_cmd); + install_element(CONFIG_NODE, &bgp_graceful_restart_stalepath_time_cmd); + install_element(CONFIG_NODE, + &no_bgp_graceful_restart_stalepath_time_cmd); + install_element(CONFIG_NODE, &bgp_graceful_restart_restart_time_cmd); + install_element(CONFIG_NODE, &no_bgp_graceful_restart_restart_time_cmd); + install_element(CONFIG_NODE, + &bgp_graceful_restart_select_defer_time_cmd); + install_element(CONFIG_NODE, + &no_bgp_graceful_restart_select_defer_time_cmd); + install_element(CONFIG_NODE, &bgp_graceful_restart_preserve_fw_cmd); + install_element(CONFIG_NODE, &no_bgp_graceful_restart_preserve_fw_cmd); + install_element(CONFIG_NODE, &bgp_graceful_restart_rib_stale_time_cmd); + install_element(CONFIG_NODE, + &no_bgp_graceful_restart_rib_stale_time_cmd); + /* Dummy commands (Currently not supported) */ install_element(BGP_NODE, &no_synchronization_cmd); install_element(BGP_NODE, &no_auto_summary_cmd); diff --git a/bgpd/bgp_vty.h b/bgpd/bgp_vty.h index addd71757d47..6d86f6ba086b 100644 --- a/bgpd/bgp_vty.h +++ b/bgpd/bgp_vty.h @@ -60,8 +60,6 @@ struct bgp; #define VTY_BGP_GR_ROUTER_DETECT(_bgp, _peer, _peer_list) \ do { \ - if (_peer->bgp->t_startup) \ - bgp_peer_gr_flags_update(_peer); \ for (ALL_LIST_ELEMENTS(_peer_list, node, nnode, peer_loop)) { \ if (CHECK_FLAG(peer_loop->flags, \ PEER_FLAG_GRACEFUL_RESTART)) \ @@ -84,30 +82,27 @@ struct bgp; } \ } while (0) -#define VTY_BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA( \ - _bgp, _peer_list, _ret) \ - do { \ - struct peer *peer_loop; \ - bool gr_router_detected = false; \ - struct listnode *node = {0}; \ - struct listnode *nnode = {0}; \ - for (ALL_LIST_ELEMENTS(_peer_list, node, nnode, peer_loop)) { \ - if (peer_loop->bgp->t_startup) \ - bgp_peer_gr_flags_update(peer_loop); \ - if (CHECK_FLAG(peer_loop->flags, \ - PEER_FLAG_GRACEFUL_RESTART)) \ - gr_router_detected = true; \ - } \ - if (gr_router_detected \ - && _bgp->present_zebra_gr_state == ZEBRA_GR_DISABLE) { \ - if (bgp_zebra_send_capabilities(_bgp, false)) \ - _ret = BGP_ERR_INVALID_VALUE; \ - } else if (!gr_router_detected \ - && _bgp->present_zebra_gr_state \ - == ZEBRA_GR_ENABLE) { \ - if (bgp_zebra_send_capabilities(_bgp, true)) \ - _ret = BGP_ERR_INVALID_VALUE; \ - } \ +#define VTY_BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(_bgp, \ + _peer_list, _ret) \ + do { \ + struct peer *peer_loop; \ + bool gr_router_detected = false; \ + struct listnode *node = { 0 }; \ + struct listnode *nnode = { 0 }; \ + for (ALL_LIST_ELEMENTS(_peer_list, node, nnode, peer_loop)) { \ + if (CHECK_FLAG(peer_loop->flags, \ + PEER_FLAG_GRACEFUL_RESTART)) \ + gr_router_detected = true; \ + } \ + if (gr_router_detected && \ + _bgp->present_zebra_gr_state == ZEBRA_GR_DISABLE) { \ + if (bgp_zebra_send_capabilities(_bgp, false)) \ + _ret = BGP_ERR_INVALID_VALUE; \ + } else if (!gr_router_detected && \ + _bgp->present_zebra_gr_state == ZEBRA_GR_ENABLE) { \ + if (bgp_zebra_send_capabilities(_bgp, true)) \ + _ret = BGP_ERR_INVALID_VALUE; \ + } \ } while (0) diff --git a/bgpd/bgp_zebra.c b/bgpd/bgp_zebra.c index 7f91e3149e0b..3508f2d341bb 100644 --- a/bgpd/bgp_zebra.c +++ b/bgpd/bgp_zebra.c @@ -3979,6 +3979,11 @@ int bgp_zebra_send_capabilities(struct bgp *bgp, bool disable) return BGP_GR_FAILURE; } + if (BGP_DEBUG(zebra, ZEBRA)) + zlog_debug("%s(%d): Sending GR capability %s to zebra", + bgp->name_pretty, bgp->vrf_id, + disable ? "disabled" : "enabled"); + /* Check if capability is already sent. If the flag force is set * send the capability since this can be initial bgp configuration */ @@ -3994,8 +3999,8 @@ int bgp_zebra_send_capabilities(struct bgp *bgp, bool disable) if (zclient_capabilities_send(ZEBRA_CLIENT_CAPABILITIES, zclient, &api) == ZCLIENT_SEND_FAILURE) { - zlog_err("%s: %s error sending capability", __func__, - bgp->name_pretty); + zlog_err("%s(%d): Error sending GR capability to zebra", + bgp->name_pretty, bgp->vrf_id); ret = BGP_GR_FAILURE; } else { if (disable) @@ -4003,9 +4008,6 @@ int bgp_zebra_send_capabilities(struct bgp *bgp, bool disable) else bgp->present_zebra_gr_state = ZEBRA_GR_ENABLE; - if (BGP_DEBUG(zebra, ZEBRA)) - zlog_debug("%s: %s send capabilty success", __func__, - bgp->name_pretty); ret = BGP_GR_SUCCESS; } return ret; diff --git a/bgpd/bgpd.c b/bgpd/bgpd.c index e42b7f5ca5c4..17d94bd57518 100644 --- a/bgpd/bgpd.c +++ b/bgpd/bgpd.c @@ -1391,9 +1391,31 @@ int bgp_global_gr_init(struct bgp *bgp) memcpy(bgp->GLOBAL_GR_FSM, local_GLOBAL_GR_FSM, sizeof(local_GLOBAL_GR_FSM)); - bgp->global_gr_present_state = GLOBAL_HELPER; + /* Inherit any BGP-wide configuration. */ + if (CHECK_FLAG(bm->flags, BM_FLAG_GR_RESTARTER)) + bgp->global_gr_present_state = GLOBAL_GR; + else if (CHECK_FLAG(bm->flags, BM_FLAG_GR_DISABLED)) + bgp->global_gr_present_state = GLOBAL_DISABLE; + else + bgp->global_gr_present_state = GLOBAL_HELPER; + + if (bm->restart_time != BGP_DEFAULT_RESTART_TIME) + bgp->restart_time = bm->restart_time; + if (bm->stalepath_time != BGP_DEFAULT_STALEPATH_TIME) + bgp->stalepath_time = bm->stalepath_time; + if (bm->select_defer_time != BGP_DEFAULT_SELECT_DEFERRAL_TIME) + bgp->select_defer_time = bm->select_defer_time; + if (bm->rib_stale_time != BGP_DEFAULT_RIB_STALE_TIME) + bgp->rib_stale_time = bm->rib_stale_time; + if (CHECK_FLAG(bm->flags, BM_FLAG_GR_PRESERVE_FWD)) + SET_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD); + bgp->present_zebra_gr_state = ZEBRA_GR_DISABLE; + if (BGP_DEBUG(graceful_restart, GRACEFUL_RESTART)) + zlog_debug("%s: Global GR state is %s", bgp->name_pretty, + print_global_gr_mode(bgp->global_gr_present_state)); + return BGP_GR_SUCCESS; } @@ -1453,9 +1475,7 @@ int bgp_peer_gr_init(struct peer *peer) { PEER_HELPER, bgp_peer_gr_action }, { PEER_GLOBAL_INHERIT, NULL } } }; - memcpy(&peer->PEER_GR_FSM, local_Peer_GR_FSM, - sizeof(local_Peer_GR_FSM)); - peer->peer_gr_present_state = PEER_GLOBAL_INHERIT; + memcpy(&peer->PEER_GR_FSM, local_Peer_GR_FSM, sizeof(local_Peer_GR_FSM)); bgp_peer_move_to_gr_mode(peer, PEER_GLOBAL_INHERIT); return BGP_GR_SUCCESS; @@ -8405,6 +8425,10 @@ void bgp_master_init(struct event_loop *master, const int buffer_size, bm->t_bgp_sync_label_manager = NULL; bm->t_bgp_start_label_manager = NULL; bm->t_bgp_zebra_route = NULL; + bm->restart_time = BGP_DEFAULT_RESTART_TIME; + bm->stalepath_time = BGP_DEFAULT_STALEPATH_TIME; + bm->select_defer_time = BGP_DEFAULT_SELECT_DEFERRAL_TIME; + bm->rib_stale_time = BGP_DEFAULT_RIB_STALE_TIME; bgp_mac_init(); /* init the rd id space. diff --git a/bgpd/bgpd.h b/bgpd/bgpd.h index 1f8cc5334986..3c5826113dec 100644 --- a/bgpd/bgpd.h +++ b/bgpd/bgpd.h @@ -163,6 +163,23 @@ struct bgp_master { uint32_t flags; #define BM_FLAG_GRACEFUL_SHUTDOWN (1 << 0) #define BM_FLAG_SEND_EXTRA_DATA_TO_ZEBRA (1 << 1) +#define BM_FLAG_MAINTENANCE_MODE (1 << 2) +#define BM_FLAG_GR_RESTARTER (1 << 3) +#define BM_FLAG_GR_DISABLED (1 << 4) +#define BM_FLAG_GR_PRESERVE_FWD (1 << 5) +#define BM_FLAG_GRACEFUL_RESTART (1 << 6) +#define BM_FLAG_GR_COMPLETE (1 << 7) + +#define BM_FLAG_GR_CONFIGURED (BM_FLAG_GR_RESTARTER | BM_FLAG_GR_DISABLED) + + /* BGP-wide graceful restart config params */ + uint32_t restart_time; + uint32_t stalepath_time; + uint32_t select_defer_time; + uint32_t rib_stale_time; + + time_t startup_time; + time_t gr_completion_time; bool terminating; /* global flag that sigint terminate seen */ @@ -293,9 +310,9 @@ struct graceful_restart_info { /* Best route select */ struct event *t_route_select; /* AFI, SAFI enabled */ - bool af_enabled[AFI_MAX][SAFI_MAX]; + bool af_enabled; /* Route update completed */ - bool route_sync[AFI_MAX][SAFI_MAX]; + bool route_sync; }; enum global_mode { @@ -547,6 +564,9 @@ struct bgp { */ enum zebra_gr_mode present_zebra_gr_state; + /* Is deferred path selection still not complete? */ + bool gr_route_sync_pending; + /* BGP Per AF flags */ uint16_t af_flags[AFI_MAX][SAFI_MAX]; #define BGP_CONFIG_DAMPENING (1 << 0) @@ -2742,6 +2762,59 @@ static inline bool bgp_in_graceful_shutdown(struct bgp *bgp) !!CHECK_FLAG(bm->flags, BM_FLAG_GRACEFUL_SHUTDOWN)); } +static inline bool bgp_in_graceful_restart(void) +{ + /* True if BGP has (re)started gracefully (based on flags + * noted at startup) and GR is not complete. + */ + return (CHECK_FLAG(bm->flags, BM_FLAG_GRACEFUL_RESTART) && + !CHECK_FLAG(bm->flags, BM_FLAG_GR_COMPLETE)); +} + +static inline bool bgp_is_graceful_restart_complete(void) +{ + /* True if BGP has (re)started gracefully (based on flags + * noted at startup) and GR is marked as complete. + */ + return (CHECK_FLAG(bm->flags, BM_FLAG_GRACEFUL_RESTART) && + CHECK_FLAG(bm->flags, BM_FLAG_GR_COMPLETE)); +} + +static inline void bgp_update_gr_completion(void) +{ + struct listnode *node, *nnode; + struct bgp *bgp; + + /* + * Check and mark GR complete. This is done when deferred + * path selection has been completed for all instances and + * route-advertisement/EOR and route-sync with zebra has + * been invoked. + */ + if (!CHECK_FLAG(bm->flags, BM_FLAG_GRACEFUL_RESTART) || + CHECK_FLAG(bm->flags, BM_FLAG_GR_COMPLETE)) + return; + + for (ALL_LIST_ELEMENTS(bm->bgp, node, nnode, bgp)) { + if (bgp->gr_route_sync_pending) + return; + } + + SET_FLAG(bm->flags, BM_FLAG_GR_COMPLETE); + bm->gr_completion_time = monotime(NULL); +} + +static inline bool bgp_gr_is_forwarding_preserved(struct bgp *bgp) +{ + /* + * Is forwarding state preserved? Based either on config + * or if BGP restarted gracefully. + * TBD: Additional AFI/SAFI based checks etc. + */ + return (CHECK_FLAG(bm->flags, BM_FLAG_GRACEFUL_RESTART) || + CHECK_FLAG(bgp->flags, BGP_FLAG_GR_PRESERVE_FWD)); +} + /* For benefit of rfapi */ extern struct peer *peer_new(struct bgp *bgp); diff --git a/doc/user/bgp.rst b/doc/user/bgp.rst index 150a915e3a88..98834c7c2349 100644 --- a/doc/user/bgp.rst +++ b/doc/user/bgp.rst @@ -92,6 +92,12 @@ be specified (:ref:`common-invocation-options`). the operator has turned off communication to zebra and is running bgpd as a complete standalone process. +.. option:: -K, --graceful_restart + + Bgpd will use this option to denote either a planned FRR graceful + restart or a bgpd-only graceful restart, and this will drive the BGP + GR restarting router procedures. + LABEL MANAGER ------------- @@ -1080,6 +1086,52 @@ Default global mode is helper and default peer per mode is inherit from global. If per peer mode is configured, the GR mode of this particular peer will override the global mode. +.. _bgp-GR-config-mode-cmd: + +BGP GR Config Mode Commands +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. clicmd:: bgp graceful-restart + + This command will enable BGP graceful restart functionality for all BGP instances. + +.. clicmd:: bgp graceful-restart-disable + + This command will disable both the functionality graceful restart and helper + mode for all BGP instances + +.. clicmd:: bgp graceful-restart select-defer-time (0-3600) + + This is command, will set deferral time to value specified. + +.. clicmd:: bgp graceful-restart rib-stale-time (1-3600) + + This is command, will set the time for which stale routes are kept in RIB. + +.. clicmd:: bgp graceful-restart restart-time (0-4095) + + Set the time to wait to delete stale routes before a BGP open message + is received. + + Using with Long-lived Graceful Restart capability, this is recommended + setting this timer to 0 and control stale routes with + ``bgp long-lived-graceful-restart stale-time``. + + Default value is 120. + +.. clicmd:: bgp graceful-restart stalepath-time (1-4095) + + This is command, will set the max time (in seconds) to hold onto + restarting peer's stale paths. + + It also controls Enhanced Route-Refresh timer. + + If this command is configured and the router does not receive a Route-Refresh EoRR + message, the router removes the stale routes from the BGP table after the timer + expires. The stale path timer is started when the router receives a Route-Refresh + BoRR message + + .. _bgp-GR-global-mode-cmd: BGP GR Global Mode Commands diff --git a/lib/libfrr.c b/lib/libfrr.c index 876efe23a826..338a7d0340f7 100644 --- a/lib/libfrr.c +++ b/lib/libfrr.c @@ -102,23 +102,25 @@ static void opt_extend(const struct optspec *os) #define OPTION_SCRIPTDIR 1009 static const struct option lo_always[] = { - {"help", no_argument, NULL, 'h'}, - {"version", no_argument, NULL, 'v'}, - {"daemon", no_argument, NULL, 'd'}, - {"module", no_argument, NULL, 'M'}, - {"profile", required_argument, NULL, 'F'}, - {"pathspace", required_argument, NULL, 'N'}, - {"vrfdefaultname", required_argument, NULL, 'o'}, - {"vty_socket", required_argument, NULL, OPTION_VTYSOCK}, - {"moduledir", required_argument, NULL, OPTION_MODULEDIR}, - {"scriptdir", required_argument, NULL, OPTION_SCRIPTDIR}, - {"log", required_argument, NULL, OPTION_LOG}, - {"log-level", required_argument, NULL, OPTION_LOGLEVEL}, - {"command-log-always", no_argument, NULL, OPTION_LOGGING}, - {"limit-fds", required_argument, NULL, OPTION_LIMIT_FDS}, - {NULL}}; + { "help", no_argument, NULL, 'h' }, + { "version", no_argument, NULL, 'v' }, + { "daemon", no_argument, NULL, 'd' }, + { "module", no_argument, NULL, 'M' }, + { "profile", required_argument, NULL, 'F' }, + { "pathspace", required_argument, NULL, 'N' }, + { "vrfdefaultname", required_argument, NULL, 'o' }, + { "graceful_restart", optional_argument, NULL, 'K' }, + { "vty_socket", required_argument, NULL, OPTION_VTYSOCK }, + { "moduledir", required_argument, NULL, OPTION_MODULEDIR }, + { "scriptdir", required_argument, NULL, OPTION_SCRIPTDIR }, + { "log", required_argument, NULL, OPTION_LOG }, + { "log-level", required_argument, NULL, OPTION_LOGLEVEL }, + { "command-log-always", no_argument, NULL, OPTION_LOGGING }, + { "limit-fds", required_argument, NULL, OPTION_LIMIT_FDS }, + { NULL } +}; static const struct optspec os_always = { - "hvdM:F:N:o:", + "hvdM:F:N:o:K::", " -h, --help Display this help and exit\n" " -v, --version Print program version\n" " -d, --daemon Runs in daemon mode\n" @@ -126,13 +128,15 @@ static const struct optspec os_always = { " -F, --profile Use specified configuration profile\n" " -N, --pathspace Insert prefix into config & socket paths\n" " -o, --vrfdefaultname Set default VRF name.\n" + " -K, --graceful_restart FRR starting in Graceful Restart mode, with optional route-cleanup timer\n" " --vty_socket Override vty socket path\n" " --moduledir Override modules directory\n" " --scriptdir Override scripts directory\n" " --log Set Logging to stdout, syslog, or file:\n" " --log-level Set Logging Level to use, debug, info, warn, etc\n" " --limit-fds Limit number of fds supported\n", - lo_always}; + lo_always +}; static bool logging_to_stdout = false; /* set when --log stdout specified */ @@ -358,6 +362,8 @@ void frr_preinit(struct frr_daemon_info *daemon, int argc, char **argv) strlcpy(frr_protonameinst, di->logname, sizeof(frr_protonameinst)); di->cli_mode = FRR_CLI_CLASSIC; + di->graceful_restart = false; + di->gr_cleanup_time = 0; /* we may be starting with extra FDs open for whatever purpose, * e.g. logging, some module, etc. Recording them here allows later @@ -520,6 +526,11 @@ static int frr_opt(int opt) di->db_file = optarg; break; #endif + case 'K': + di->graceful_restart = true; + if (optarg) + di->gr_cleanup_time = atoi(optarg); + break; case 'C': if (di->flags & FRR_NO_SPLIT_CONFIG) return 1; diff --git a/lib/libfrr.h b/lib/libfrr.h index 77d70448a9a1..db9cfbcb1f2b 100644 --- a/lib/libfrr.h +++ b/lib/libfrr.h @@ -118,6 +118,8 @@ struct frr_daemon_info { bool dryrun; bool daemon_mode; bool terminal; + bool graceful_restart; + int gr_cleanup_time; enum frr_cli_mode cli_mode; struct event *read_in; diff --git a/tests/topotests/bgp_gr_functionality_topo1/test_bgp_gr_functionality_topo1-4.py b/tests/topotests/bgp_gr_functionality_topo1/test_bgp_gr_functionality_topo1-4.py index 31aaa0b8a6df..de4b94032c06 100644 --- a/tests/topotests/bgp_gr_functionality_topo1/test_bgp_gr_functionality_topo1-4.py +++ b/tests/topotests/bgp_gr_functionality_topo1/test_bgp_gr_functionality_topo1-4.py @@ -81,6 +81,8 @@ import sys import time import pytest +import functools +import json # Save the Current Working Directory to find configuration files. CWD = os.path.dirname(os.path.realpath(__file__)) @@ -91,6 +93,7 @@ # Import topogen and topotest helpers from lib.topogen import Topogen, get_topogen from lib.topolog import logger +from lib import topotest # Required to instantiate the topology builder class. @@ -1680,6 +1683,304 @@ def BGP_GR_TC_52_p1(request): write_test_footer(tc_name) +def test_BGP_GR_TC_53_p1(request): + """ + Test Objective : Peer-level inherit from BGP wide Restarting + Global Mode : GR Restart + PerPeer Mode : None + GR Mode effective : GR Restart + + """ + + tgen = get_topogen() + tc_name = request.node.name + write_test_header(tc_name) + + # Check router status + check_router_status(tgen) + + # Don't run this test if we have any failure. + if tgen.routers_have_failure(): + pytest.skip(tgen.errors) + + # Creating configuration from JSON + reset_config_on_routers(tgen) + + step("Configure R1 as GR restarting node in global level") + + input_dict = { + "r1": {"graceful-restart": {"graceful-restart": True}}, + "r2": {"graceful-restart": {"graceful-restart-helper": True}}, + } + + output = tgen.gears["r1"].vtysh_cmd( + """ + configure terminal + bgp graceful-restart + ! + """ + ) + + step("Verify that R2 receives GR restarting capabilities" " from R1") + + for addr_type in ADDR_TYPES: + result = verify_graceful_restart( + tgen, topo, addr_type, input_dict, dut="r1", peer="r2" + ) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + result = verify_graceful_restart( + tgen, topo, addr_type, input_dict, dut="r2", peer="r1" + ) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + + for addr_type in ADDR_TYPES: + dut = "r1" + peer = "r2" + protocol = "bgp" + next_hop = next_hop_per_address_family( + tgen, dut, peer, addr_type, NEXT_HOP_IP_2 + ) + input_topo = {"r2": topo["routers"]["r2"]} + result = verify_rib(tgen, addr_type, dut, input_topo, next_hop, protocol) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + + dut = "r2" + peer = "r1" + next_hop = next_hop_per_address_family( + tgen, dut, peer, addr_type, NEXT_HOP_IP_1 + ) + input_topo = {"r1": topo["routers"]["r1"]} + result = verify_bgp_rib(tgen, addr_type, dut, input_topo, next_hop) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + result = verify_rib(tgen, addr_type, dut, input_topo, next_hop, protocol) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + + step("Kill BGPd on router R1") + + kill_router_daemons(tgen, "r1", ["bgpd"]) + + step( + "Verify that R1 keeps the stale entries in FIB and R2 keeps stale entries in RIB & FIB" + ) + + for addr_type in ADDR_TYPES: + dut = "r1" + peer = "r2" + protocol = "bgp" + next_hop = next_hop_per_address_family( + tgen, dut, peer, addr_type, NEXT_HOP_IP_2 + ) + input_topo = {"r2": topo["routers"]["r2"]} + result = verify_rib(tgen, addr_type, dut, input_topo, next_hop, protocol) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + + dut = "r2" + peer = "r1" + next_hop = next_hop_per_address_family( + tgen, dut, peer, addr_type, NEXT_HOP_IP_1 + ) + input_topo = {"r1": topo["routers"]["r1"]} + result = verify_bgp_rib(tgen, addr_type, dut, input_topo, next_hop) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + result = verify_rib(tgen, addr_type, dut, input_topo, next_hop, protocol) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + + # Configure graceful-restart-disable at config global level verify that the functionality works + step("Bring up BGP on R1 and configure graceful-restart-disable") + + start_router_daemons(tgen, "r1", ["bgpd"]) + + input_dict = { + "r1": {"graceful-restart": {"graceful-restart-disable": True}}, + "r2": {"graceful-restart": {"graceful-restart-helper": True}}, + } + + output = tgen.gears["r1"].vtysh_cmd( + """ + configure terminal + bgp graceful-restart-disable + ! + """ + ) + + step("Verify on R2 that R1 does't advertise any GR capabilities") + + for addr_type in ADDR_TYPES: + result = verify_graceful_restart( + tgen, topo, addr_type, input_dict, dut="r1", peer="r2" + ) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + result = verify_graceful_restart( + tgen, topo, addr_type, input_dict, dut="r2", peer="r1" + ) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + + for addr_type in ADDR_TYPES: + dut = "r1" + peer = "r2" + protocol = "bgp" + next_hop = next_hop_per_address_family( + tgen, dut, peer, addr_type, NEXT_HOP_IP_2 + ) + input_topo = {"r2": topo["routers"]["r2"]} + result = verify_rib(tgen, addr_type, dut, input_topo, next_hop, protocol) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + + dut = "r2" + peer = "r1" + next_hop = next_hop_per_address_family( + tgen, dut, peer, addr_type, NEXT_HOP_IP_1 + ) + input_topo = {"r1": topo["routers"]["r1"]} + result = verify_bgp_rib(tgen, addr_type, dut, input_topo, next_hop) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + result = verify_rib(tgen, addr_type, dut, input_topo, next_hop, protocol) + assert result is True, "Testcase {} :Failed \n Error {}".format(tc_name, result) + + step("Kill BGP on R1") + + kill_router_daemons(tgen, "r1", ["bgpd"]) + + step("Verify on R2 and R1 that none of the routers keep stale entries") + + for addr_type in ADDR_TYPES: + dut = "r1" + peer = "r2" + protocol = "bgp" + next_hop = next_hop_per_address_family( + tgen, dut, peer, addr_type, NEXT_HOP_IP_2 + ) + input_topo = {"r2": topo["routers"]["r2"]} + result = verify_rib( + tgen, addr_type, dut, input_topo, next_hop, protocol, expected=False + ) + assert result is not True, ( + "Testcase {} : Failed \n " + "Expected: Routes should not be present in {} FIB \n " + "Found: {}".format(tc_name, dut, result) + ) + + dut = "r2" + peer = "r1" + next_hop = next_hop_per_address_family( + tgen, dut, peer, addr_type, NEXT_HOP_IP_1 + ) + input_topo = {"r1": topo["routers"]["r1"]} + result = verify_bgp_rib( + tgen, addr_type, dut, input_topo, next_hop, expected=False + ) + assert result is not True, ( + "Testcase {} : Failed \n " + "Expected: Routes should not be present in {} BGP RIB \n " + "Found: {}".format(tc_name, dut, result) + ) + + result = verify_rib( + tgen, addr_type, dut, input_topo, next_hop, protocol, expected=False + ) + assert result is not True, ( + "Testcase {} : Failed \n " + "Expected: Routes should not be present in {} FIB \n " + "Found: {}".format(tc_name, dut, result) + ) + + step( + "Bring up BGP on R1, enable GR and configure bgp graceful-restart restart-time at global level" + ) + + start_router_daemons(tgen, "r1", ["bgpd"]) + + output = tgen.gears["r1"].vtysh_cmd( + """ + configure terminal + no bgp graceful-restart-disable + bgp graceful-restart + bgp graceful-restart stalepath-time 420 + bgp graceful-restart restart-time 240 + bgp graceful-restart select-defer-time 420 + ! + """ + ) + + step("Verify on R2 that R1 sent the updated GR restart-time") + + def _bgp_check_if_gr_restart_time_was_updated(): + output = json.loads(tgen.gears["r2"].vtysh_cmd("show bgp neighbor json")) + + expected = { + "192.168.0.1": { + "gracefulRestartInfo": { + "localGrMode": "Helper*", + "remoteGrMode": "Restart", + "timers": { + "configuredRestartTimer": 120, + "receivedRestartTimer": 240, + }, + }, + }, + "fd00::1": { + "gracefulRestartInfo": { + "localGrMode": "Helper*", + "remoteGrMode": "Restart", + "timers": { + "configuredRestartTimer": 120, + "receivedRestartTimer": 240, + }, + }, + }, + } + return topotest.json_cmp(output, expected) + + test_func = functools.partial( + _bgp_check_if_gr_restart_time_was_updated, + ) + _, result = topotest.run_and_expect(test_func, None, count=30, wait=1) + assert result is None, "R2 did not receive the updated GR restart-time of 240s" + + def _bgp_check_if_gr_timer_on_restarting_node_was_updated(): + output = json.loads(tgen.gears["r1"].vtysh_cmd("show bgp neighbor json")) + + expected = { + "192.168.0.2": { + "gracefulRestartInfo": { + "localGrMode": "Restart*", + "remoteGrMode": "Helper", + "timers": { + "configuredRestartTimer": 240, + "receivedRestartTimer": 120, + }, + "ipv4Unicast": { + "timers": {"stalePathTimer": 420, "selectionDeferralTimer": 420} + }, + }, + }, + "fd00::2": { + "gracefulRestartInfo": { + "localGrMode": "Restart*", + "remoteGrMode": "Helper", + "timers": { + "configuredRestartTimer": 240, + "receivedRestartTimer": 120, + }, + "ipv6Unicast": { + "timers": {"stalePathTimer": 420, "selectionDeferralTimer": 420} + }, + }, + }, + } + return topotest.json_cmp(output, expected) + + test_func = functools.partial( + _bgp_check_if_gr_timer_on_restarting_node_was_updated, + ) + _, result = topotest.run_and_expect(test_func, None, count=30, wait=1) + assert ( + result is None + ), "R1 did not update the GR select-deferral and stale-path timer to 420s" + + write_test_footer(tc_name) + + if __name__ == "__main__": args = ["-s"] + sys.argv[1:] sys.exit(pytest.main(args)) diff --git a/tests/topotests/bgp_gr_notification/test_bgp_gr_notification.py b/tests/topotests/bgp_gr_notification/test_bgp_gr_notification.py index 16459a25a33d..5d8338d6eb2a 100644 --- a/tests/topotests/bgp_gr_notification/test_bgp_gr_notification.py +++ b/tests/topotests/bgp_gr_notification/test_bgp_gr_notification.py @@ -195,16 +195,16 @@ def _bgp_clear_r1_and_shutdown(): step("Reset and shutdown R1") _bgp_clear_r1_and_shutdown() - step("Check if Hard Reset notification wasn't sent from R2") - test_func = functools.partial(_bgp_check_hard_reset) - _, result = topotest.run_and_expect(test_func, None, count=60, wait=0.5) - assert result is None, "Failed to send Administrative Reset notification from R2" - step("Check if stale routes are retained on R1") test_func = functools.partial(_bgp_check_gr_notification_stale) _, result = topotest.run_and_expect(test_func, None, count=60, wait=0.5) assert result is None, "Failed to see retained stale routes on R1" + step("Check if Hard Reset notification wasn't sent from R2") + test_func = functools.partial(_bgp_check_hard_reset) + _, result = topotest.run_and_expect(test_func, None, count=60, wait=0.5) + assert result is None, "Failed to send Administrative Reset notification from R2" + if __name__ == "__main__": args = ["-s"] + sys.argv[1:] diff --git a/tests/topotests/lib/bgp.py b/tests/topotests/lib/bgp.py index 4250c405f3dc..bcd1c748120f 100644 --- a/tests/topotests/lib/bgp.py +++ b/tests/topotests/lib/bgp.py @@ -3266,27 +3266,43 @@ def verify_graceful_restart( lmode = None rmode = None + # Local GR mode - if "address_family" in input_dict[dut]["bgp"]: - bgp_neighbors = input_dict[dut]["bgp"]["address_family"][addr_type][ - "unicast" - ]["neighbor"][peer]["dest_link"] + if "bgp" not in input_dict[dut] and "graceful-restart" in input_dict[dut]: + if ( + "graceful-restart" in input_dict[dut]["graceful-restart"] + and input_dict[dut]["graceful-restart"]["graceful-restart"] + ): + lmode = "Restart*" + elif ( + "graceful-restart-disable" in input_dict[dut]["graceful-restart"] + and input_dict[dut]["graceful-restart"]["graceful-restart-disable"] + ): + lmode = "Disable*" + else: + lmode = "Helper*" - for dest_link, data in bgp_neighbors.items(): - if ( - "graceful-restart-helper" in data - and data["graceful-restart-helper"] - ): - lmode = "Helper" - elif "graceful-restart" in data and data["graceful-restart"]: - lmode = "Restart" - elif ( - "graceful-restart-disable" in data - and data["graceful-restart-disable"] - ): - lmode = "Disable" - else: - lmode = None + if lmode is None: + if "address_family" in input_dict[dut]["bgp"]: + bgp_neighbors = input_dict[dut]["bgp"]["address_family"][addr_type][ + "unicast" + ]["neighbor"][peer]["dest_link"] + + for dest_link, data in bgp_neighbors.items(): + if ( + "graceful-restart-helper" in data + and data["graceful-restart-helper"] + ): + lmode = "Helper" + elif "graceful-restart" in data and data["graceful-restart"]: + lmode = "Restart" + elif ( + "graceful-restart-disable" in data + and data["graceful-restart-disable"] + ): + lmode = "Disable" + else: + lmode = None if lmode is None: if "graceful-restart" in input_dict[dut]["bgp"]: @@ -3314,7 +3330,11 @@ def verify_graceful_restart( return True # Remote GR mode - if "address_family" in input_dict[peer]["bgp"]: + + if ( + "bgp" in input_dict[peer] + and "address_family" in input_dict[peer]["bgp"] + ): bgp_neighbors = input_dict[peer]["bgp"]["address_family"][addr_type][ "unicast" ]["neighbor"][dut]["dest_link"] @@ -3336,7 +3356,10 @@ def verify_graceful_restart( rmode = None if rmode is None: - if "graceful-restart" in input_dict[peer]["bgp"]: + if ( + "bgp" in input_dict[peer] + and "graceful-restart" in input_dict[peer]["bgp"] + ): if ( "graceful-restart" in input_dict[peer]["bgp"]["graceful-restart"] @@ -3355,6 +3378,27 @@ def verify_graceful_restart( rmode = "Disable" else: rmode = "Helper" + + if rmode is None: + if ( + "bgp" not in input_dict[peer] + and "graceful-restart" in input_dict[peer] + ): + if ( + "graceful-restart" in input_dict[peer]["graceful-restart"] + and input_dict[peer]["graceful-restart"]["graceful-restart"] + ): + rmode = "Restart" + elif ( + "graceful-restart-disable" + in input_dict[peer]["graceful-restart"] + and input_dict[peer]["graceful-restart"][ + "graceful-restart-disable" + ] + ): + rmode = "Disable" + else: + rmode = "Helper" else: rmode = "Helper" diff --git a/zebra/main.c b/zebra/main.c index ea1e1cbdbbd8..687da70cabce 100644 --- a/zebra/main.c +++ b/zebra/main.c @@ -65,8 +65,6 @@ struct mgmt_be_client *mgmt_be_client; /* Route retain mode flag. */ int retain_mode = 0; -int graceful_restart; - /* Receive buffer size for kernel control sockets */ #define RCVBUFSIZE_MIN 4194304 #ifdef HAVE_NETLINK @@ -88,7 +86,6 @@ const struct option longopts[] = { { "socket", required_argument, NULL, 'z' }, { "ecmp", required_argument, NULL, 'e' }, { "retain", no_argument, NULL, 'r' }, - { "graceful_restart", required_argument, NULL, 'K' }, { "asic-offload", optional_argument, NULL, OPTION_ASIC_OFFLOAD }, { "v6-with-v4-nexthops", no_argument, NULL, OPTION_V6_WITH_V4_NEXTHOP }, #ifdef HAVE_NETLINK @@ -96,7 +93,7 @@ const struct option longopts[] = { { "nl-bufsize", required_argument, NULL, 's' }, { "v6-rr-semantics", no_argument, NULL, OPTION_V6_RR_SEMANTICS }, #endif /* HAVE_NETLINK */ - {"routing-table", optional_argument, NULL, 'R'}, + { "routing-table", optional_argument, NULL, 'R' }, { 0 } }; @@ -326,7 +323,6 @@ int main(int argc, char **argv) bool v6_with_v4_nexthop = false; bool notify_on_ack = true; - graceful_restart = 0; vrf_configure_backend(VRF_BACKEND_VRF_LITE); frr_preinit(&zebra_di, argc, argv); @@ -342,7 +338,6 @@ int main(int argc, char **argv) " -z, --socket Set path of zebra socket\n" " -e, --ecmp Specify ECMP to use.\n" " -r, --retain When program terminates, retain added route by zebra.\n" - " -K, --graceful_restart Graceful restart at the kernel level, timer in seconds for expiration\n" " -A, --asic-offload FRR is interacting with an asic underneath the linux kernel\n" " --v6-with-v4-nexthops Underlying dataplane supports v6 routes with v4 nexthops" #ifdef HAVE_NETLINK @@ -352,8 +347,7 @@ int main(int argc, char **argv) #else " -s, Set kernel socket receive buffer size\n" #endif /* HAVE_NETLINK */ - " -R, --routing-table Set kernel routing table\n" - ); + " -R, --routing-table Set kernel routing table\n"); while (1) { int opt = frr_getopt(argc, argv, NULL); @@ -397,9 +391,6 @@ int main(int argc, char **argv) case 'r': retain_mode = 1; break; - case 'K': - graceful_restart = atoi(optarg); - break; case 's': rcvbufsize = atoi(optarg); if (rcvbufsize < RCVBUFSIZE_MIN) @@ -488,11 +479,25 @@ int main(int argc, char **argv) * Clean up zebra-originated routes. The requests will be sent to OS * immediately, so originating PID in notifications from kernel * will be equal to the current getpid(). To know about such routes, - * we have to have route_read() called before. + * we have to have route_read() called before. + * If FRR is gracefully restarting, we either wait for clients + * (e.g., BGP) to signal GR is complete else we wait for specified + * duration. */ zrouter.startup_time = monotime(NULL); - event_add_timer(zrouter.master, rib_sweep_route, NULL, graceful_restart, - &zrouter.sweeper); + zrouter.rib_sweep_time = 0; + zrouter.graceful_restart = zebra_di.graceful_restart; + if (!zrouter.graceful_restart) + event_add_timer(zrouter.master, rib_sweep_route, NULL, 0, NULL); + else { + int gr_cleanup_time; + + gr_cleanup_time = zebra_di.gr_cleanup_time + ? zebra_di.gr_cleanup_time + : ZEBRA_GR_DEFAULT_RIB_SWEEP_TIME; + event_add_timer(zrouter.master, rib_sweep_route, NULL, + gr_cleanup_time, &zrouter.t_rib_sweep); + } /* Needed for BSD routing socket. */ pid = getpid(); diff --git a/zebra/rib.h b/zebra/rib.h index a721f4bac456..84ea766c4733 100644 --- a/zebra/rib.h +++ b/zebra/rib.h @@ -622,10 +622,10 @@ static inline struct nexthop_group *rib_get_fib_backup_nhg( } extern void zebra_gr_process_client(afi_t afi, vrf_id_t vrf_id, uint8_t proto, - uint8_t instance); + uint8_t instance, time_t restart_time); extern int rib_add_gr_run(afi_t afi, vrf_id_t vrf_id, uint8_t proto, - uint8_t instance); + uint8_t instance, time_t restart_time); extern void zebra_vty_init(void); diff --git a/zebra/zapi_msg.c b/zebra/zapi_msg.c index 2a1eea9594ee..654ade806355 100644 --- a/zebra/zapi_msg.c +++ b/zebra/zapi_msg.c @@ -2414,6 +2414,7 @@ static void zsend_capabilities(struct zserv *client, struct zebra_vrf *zvrf) stream_putl(s, zrouter.multipath_num); stream_putc(s, zebra_mlag_get_role()); stream_putc(s, zrouter.v6_with_v4_nexthop); + stream_putc(s, zrouter.graceful_restart); stream_putw_at(s, 0, stream_get_endp(s)); zserv_send_message(client, s); } diff --git a/zebra/zebra_gr.c b/zebra/zebra_gr.c index cee66cc05545..d84f680e2288 100644 --- a/zebra/zebra_gr.c +++ b/zebra/zebra_gr.c @@ -103,6 +103,7 @@ static struct client_gr_info *zebra_gr_client_info_create(struct zserv *client) info->stale_client_ptr = client; TAILQ_INSERT_TAIL(&(client->gr_info_queue), info, gr_info); + info->client_ptr = client; return info; } @@ -290,6 +291,7 @@ struct zebra_gr_afi_clean { afi_t afi; uint8_t proto; uint8_t instance; + time_t restart_time; struct event *t_gac; }; @@ -420,7 +422,7 @@ void zread_client_capabilities(ZAPI_HANDLER_ARGS) * Schedule for after anything already in the meta Q */ rib_add_gr_run(api.afi, api.vrf_id, client->proto, - client->instance); + client->instance, client->restart_time); zebra_gr_process_client_stale_routes(client, info); break; case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING: @@ -455,7 +457,11 @@ static void zebra_gr_route_stale_delete_timer_expiry(struct event *thread) struct zserv *client; struct vrf *vrf = vrf_lookup_by_id(info->vrf_id); - client = (struct zserv *)info->stale_client_ptr; + info->t_stale_removal = NULL; + if (zrouter.graceful_restart) + client = (struct zserv *)info->client_ptr; + else + client = (struct zserv *)info->stale_client_ptr; cnt = zebra_gr_delete_stale_routes(info); @@ -486,16 +492,24 @@ static void zebra_gr_route_stale_delete_timer_expiry(struct event *thread) * * Returns true when a node is deleted else false */ -static bool zebra_gr_process_route_entry(struct zserv *client, - struct route_node *rn, - struct route_entry *re) +static bool zebra_gr_process_route_entry(struct route_node *rn, + struct route_entry *re, + time_t compare_time, uint8_t proto) { + struct nexthop *nexthop; + char buf[PREFIX2STR_BUFFER]; + /* If the route is not refreshed after restart, delete the entry */ - if (re->uptime < client->restart_time) { - if (IS_ZEBRA_DEBUG_RIB) - zlog_debug("%s: Client %s stale route %pFX is deleted", - __func__, zebra_route_string(client->proto), - &rn->p); + if (re->uptime < compare_time) { + if (IS_ZEBRA_DEBUG_RIB) { + prefix2str(&rn->p, buf, sizeof(buf)); + zlog_debug("%s: Client %s stale route %s is deleted", + __func__, zebra_route_string(proto), buf); + } + SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED); + for (ALL_NEXTHOPS(re->nhe->nhg, nexthop)) + SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB); + rib_delnode(rn, re); return true; @@ -532,8 +546,9 @@ static void zebra_gr_delete_stale_route_table_afi(struct event *event) if (re->type == gac->proto && re->instance == gac->instance && - zebra_gr_process_route_entry( - gac->info->stale_client_ptr, rn, re)) + zebra_gr_process_route_entry(rn, re, + gac->restart_time, + gac->proto)) n++; /* If the max route count is reached @@ -567,28 +582,42 @@ static int32_t zebra_gr_delete_stale_route(struct client_gr_info *info, uint8_t proto; uint16_t instance; struct zserv *s_client; + struct zserv *client; + time_t restart_time = time(NULL); - s_client = info->stale_client_ptr; - if (s_client == NULL) { - LOG_GR("%s: Stale client %s(%u) not present", __func__, - zvrf->vrf->name, zvrf->vrf->vrf_id); + if ((info == NULL) || (zvrf == NULL)) return -1; - } - proto = s_client->proto; - instance = s_client->instance; + if (zrouter.graceful_restart) { + client = info->client_ptr; + if (client == NULL) { + LOG_GR("%s: client not present", __func__); + return -1; + } + proto = client->proto; + instance = client->instance; + restart_time = zrouter.startup_time; + } else { + s_client = info->stale_client_ptr; + if (s_client == NULL) { + LOG_GR("%s: Stale client not present", __func__); + return -1; + } + proto = s_client->proto; + instance = s_client->instance; + restart_time = s_client->restart_time; + } LOG_GR("%s: Client %s %s(%u) stale routes are being deleted", __func__, zebra_route_string(proto), zvrf->vrf->name, zvrf->vrf->vrf_id); /* Process routes for all AFI */ for (afi = AFI_IP; afi < AFI_MAX; afi++) { - /* * Schedule for immediately after anything in the * meta-Q */ - rib_add_gr_run(afi, info->vrf_id, proto, instance); + rib_add_gr_run(afi, info->vrf_id, proto, instance, restart_time); } return 0; } @@ -641,12 +670,13 @@ static void zebra_gr_process_client_stale_routes(struct zserv *client, /* * Route update completed for all AFI, SAFI - * Cancel the stale timer, routes are already being processed + * Also perform the cleanup if FRR itself is gracefully restarting. */ - if (info->t_stale_removal) { + info->route_sync_done_time = monotime(NULL); + if (info->t_stale_removal || zrouter.graceful_restart) { struct vrf *vrf = vrf_lookup_by_id(info->vrf_id); - LOG_GR("%s: Client %s canceled stale delete timer vrf %s(%d)", + LOG_GR("%s: Client %s route update complete for all AFI/SAFI in vrf %s(%d)", __func__, zebra_route_string(client->proto), VRF_LOGNAME(vrf), info->vrf_id); EVENT_OFF(info->t_stale_removal); @@ -654,7 +684,7 @@ static void zebra_gr_process_client_stale_routes(struct zserv *client, } void zebra_gr_process_client(afi_t afi, vrf_id_t vrf_id, uint8_t proto, - uint8_t instance) + uint8_t instance, time_t restart_time) { struct zserv *client = zserv_find_client(proto, instance); struct client_gr_info *info = NULL; @@ -676,6 +706,7 @@ void zebra_gr_process_client(afi_t afi, vrf_id_t vrf_id, uint8_t proto, gac->afi = afi; gac->proto = proto; gac->instance = instance; + gac->restart_time = restart_time; event_add_event(zrouter.master, zebra_gr_delete_stale_route_table_afi, gac, 0, &gac->t_gac); diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c index 5b95d8668af0..b176ea2fe6c5 100644 --- a/zebra/zebra_rib.c +++ b/zebra/zebra_rib.c @@ -3159,6 +3159,7 @@ struct meta_q_gr_run { vrf_id_t vrf_id; uint8_t proto; uint8_t instance; + time_t restart_time; }; static void process_subq_gr_run(struct listnode *lnode) @@ -3166,7 +3167,7 @@ static void process_subq_gr_run(struct listnode *lnode) struct meta_q_gr_run *gr_run = listgetdata(lnode); zebra_gr_process_client(gr_run->afi, gr_run->vrf_id, gr_run->proto, - gr_run->instance); + gr_run->instance, gr_run->restart_time); XFREE(MTYPE_WQ_WRAPPER, gr_run); } @@ -4266,7 +4267,8 @@ static int rib_meta_queue_early_route_add(struct meta_queue *mq, void *data) return 0; } -int rib_add_gr_run(afi_t afi, vrf_id_t vrf_id, uint8_t proto, uint8_t instance) +int rib_add_gr_run(afi_t afi, vrf_id_t vrf_id, uint8_t proto, uint8_t instance, + time_t restart_time) { struct meta_q_gr_run *gr_run; @@ -4276,6 +4278,7 @@ int rib_add_gr_run(afi_t afi, vrf_id_t vrf_id, uint8_t proto, uint8_t instance) gr_run->proto = proto; gr_run->vrf_id = vrf_id; gr_run->instance = instance; + gr_run->restart_time = restart_time; return mq_add_handler(gr_run, rib_meta_queue_gr_run_add); } @@ -4694,6 +4697,10 @@ void rib_sweep_route(struct event *t) struct vrf *vrf; struct zebra_vrf *zvrf; + zrouter.rib_sweep_time = monotime(NULL); + /* TODO: Change to debug */ + zlog_info("Sweeping the RIB for stale routes..."); + RB_FOREACH (vrf, vrf_id_head, &vrfs_by_id) { if ((zvrf = vrf->info) == NULL) continue; diff --git a/zebra/zebra_router.c b/zebra/zebra_router.c index 3fd4e6eb1f63..8d6b2f347651 100644 --- a/zebra/zebra_router.c +++ b/zebra/zebra_router.c @@ -238,7 +238,7 @@ void zebra_router_terminate(void) { struct zebra_router_table *zrt, *tmp; - EVENT_OFF(zrouter.sweeper); + EVENT_OFF(zrouter.t_rib_sweep); RB_FOREACH_SAFE (zrt, zebra_router_table_head, &zrouter.tables, tmp) zebra_router_free_table(zrt); diff --git a/zebra/zebra_router.h b/zebra/zebra_router.h index 304170743994..c86c6be1ef56 100644 --- a/zebra/zebra_router.h +++ b/zebra/zebra_router.h @@ -191,10 +191,16 @@ struct zebra_router { enum multicast_mode ipv4_multicast_mode; /* - * Time for when we sweep the rib from old routes + * zebra start time and time of sweeping RIB of old routes */ time_t startup_time; - struct event *sweeper; + time_t rib_sweep_time; + + /* FRR fast/graceful restart info */ + bool graceful_restart; + int gr_cleanup_time; +#define ZEBRA_GR_DEFAULT_RIB_SWEEP_TIME 500 + struct event *t_rib_sweep; /* * The hash of nexthop groups associated with this router diff --git a/zebra/zebra_vty.c b/zebra/zebra_vty.c index c31218a7c367..858e503031a4 100644 --- a/zebra/zebra_vty.c +++ b/zebra/zebra_vty.c @@ -3825,6 +3825,20 @@ DEFUN (show_zebra, struct vrf *vrf; struct ttable *table = ttable_new(&ttable_styles[TTSTYLE_BLANK]); char *out; + char timebuf[MONOTIME_STRLEN]; + + time_to_string(zrouter.startup_time, timebuf); + vty_out(vty, "Zebra started%s at time %s", + zrouter.graceful_restart ? " gracefully" : "", timebuf); + + if (zrouter.t_rib_sweep) + vty_out(vty, + "Zebra RIB sweep timer running, remaining time %lds\n", + event_timer_remain_second(zrouter.t_rib_sweep)); + else { + time_to_string(zrouter.rib_sweep_time, timebuf); + vty_out(vty, "Zebra RIB sweep happened at %s", timebuf); + } ttable_rowseps(table, 0, BOTTOM, true, '-'); ttable_add_row(table, "OS|%s(%s)", cmd_system_get(), cmd_release_get()); diff --git a/zebra/zserv.c b/zebra/zserv.c index 27668534ee99..a731f7f278fd 100644 --- a/zebra/zserv.c +++ b/zebra/zserv.c @@ -1031,6 +1031,7 @@ static char *zserv_time_buf(time_t *time1, char *buf, int buflen) /* Display client info details */ static void zebra_show_client_detail(struct vty *vty, struct zserv *client) { + struct client_gr_info *info; char cbuf[ZEBRA_TIME_BUF], rbuf[ZEBRA_TIME_BUF]; char wbuf[ZEBRA_TIME_BUF], nhbuf[ZEBRA_TIME_BUF], mbuf[ZEBRA_TIME_BUF]; time_t connect_time, last_read_time, last_write_time; @@ -1125,6 +1126,45 @@ static void zebra_show_client_detail(struct vty *vty, struct zserv *client) vty_out(vty, "ES-EVI %-12u%-12u%-12u\n", client->local_es_evi_add_cnt, 0, client->local_es_evi_del_cnt); vty_out(vty, "Errors: %u\n", client->error_cnt); + + TAILQ_FOREACH (info, &client->gr_info_queue, gr_info) { + afi_t afi; + bool route_sync_done = true; + char timebuf[MONOTIME_STRLEN]; + + vty_out(vty, "VRF : %s\n", vrf_id_to_name(info->vrf_id)); + vty_out(vty, "Capabilities : "); + switch (info->capabilities) { + case ZEBRA_CLIENT_GR_CAPABILITIES: + vty_out(vty, "Graceful Restart\n"); + break; + case ZEBRA_CLIENT_ROUTE_UPDATE_COMPLETE: + case ZEBRA_CLIENT_ROUTE_UPDATE_PENDING: + case ZEBRA_CLIENT_GR_DISABLE: + case ZEBRA_CLIENT_RIB_STALE_TIME: + vty_out(vty, "None\n"); + break; + } + for (afi = AFI_IP; afi < AFI_MAX; afi++) { + if (info->af_enabled[afi]) { + if (info->route_sync[afi]) + vty_out(vty, + "AFI %d enabled, route sync DONE\n", + afi); + else { + vty_out(vty, + "AFI %d enabled, route sync NOT DONE\n", + afi); + route_sync_done = false; + } + } + } + if (route_sync_done) { + time_to_string(info->route_sync_done_time, timebuf); + vty_out(vty, "Route sync finished at %s", timebuf); + } + } + vty_out(vty, "Input Fifo: %zu:%zu Output Fifo: %zu:%zu\n", client->ibuf_fifo->count, client->ibuf_fifo->max_count, client->obuf_fifo->count, client->obuf_fifo->max_count); diff --git a/zebra/zserv.h b/zebra/zserv.h index 57d673060fe7..87d2b4adbf81 100644 --- a/zebra/zserv.h +++ b/zebra/zserv.h @@ -64,6 +64,8 @@ struct client_gr_info { /* Book keeping */ void *stale_client_ptr; struct event *t_stale_removal; + void *client_ptr; + time_t route_sync_done_time; TAILQ_ENTRY(client_gr_info) gr_info; };