diff --git a/ansible/library/check_bgp_ipv6_routes_converged.py b/ansible/library/check_bgp_ipv6_routes_converged.py index ef36f44cf80..2a77d7666a5 100644 --- a/ansible/library/check_bgp_ipv6_routes_converged.py +++ b/ansible/library/check_bgp_ipv6_routes_converged.py @@ -10,6 +10,11 @@ import base64 +# Constants +INTERFACE_COMMAND_TEMPLATE = "sudo config interface {action} {target}" +BGP_COMMAND_TEMPLATE = "sudo config bgp {action} {target}" + + def get_bgp_ipv6_routes(module): cmd = "docker exec bgp vtysh -c 'show ipv6 route bgp json'" rc, out, err = module.run_command(cmd, executable='/bin/bash', use_unsafe_shell=True) @@ -18,6 +23,40 @@ def get_bgp_ipv6_routes(module): return json.loads(out) +def perform_action(module, action, connection_type, targets, all_neighbors): + """ + Perform actions (shutdown/startup) on BGP sessions or interfaces. + """ + # Action on BGP sessions + if connection_type == "bgp_sessions": + if all_neighbors: + cmd = BGP_COMMAND_TEMPLATE.format(action=action, target="all") + execute_command(module, cmd) + else: + for session in targets: + target_session = "neighbor " + session + cmd = BGP_COMMAND_TEMPLATE.format(action=action, target=target_session) + execute_command(module, cmd) + logging.info(f"BGP sessions {action} completed.") + # Action on Interfaces + elif connection_type == "ports": + ports_str = ",".join(targets) + cmd = INTERFACE_COMMAND_TEMPLATE.format(action=action, target=ports_str) + execute_command(module, cmd) + logging.info(f"Interfaces {action} completed.") + else: + logging.info("No valid connection type provided for %s.", action) + + +def execute_command(module, cmd): + """Helper function to execute shell commands.""" + logging.info("Running command: %s", cmd) + rc, out, err = module.run_command(cmd, executable="/bin/bash", use_unsafe_shell=True) + if rc != 0: + module.fail_json(msg=f"Command failed: {err}") + logging.info("Command completed successfully.") + + def compare_routes(running_routes, expected_routes): expected_set = set(expected_routes.keys()) running_set = set(running_routes.keys()) @@ -48,7 +87,9 @@ def main(): module = AnsibleModule( argument_spec=dict( expected_routes=dict(required=True, type='str'), - shutdown_ports=dict(required=True, type='list', elements='str'), + shutdown_connections=dict(required=True, type='list', elements='str'), + connection_type=dict(required=False, type='str', choices=['ports', 'bgp_sessions', 'none'], default='none'), + shutdown_all_connections=dict(required=False, type='bool', default=False), timeout=dict(required=False, type='int', default=300), interval=dict(required=False, type='int', default=1), log_path=dict(required=False, type='str', default='/tmp'), @@ -71,7 +112,9 @@ def main(): else: expected_routes = json.loads(module.params['expected_routes']) - shutdown_ports = module.params['shutdown_ports'] + shutdown_connections = module.params.get('shutdown_connections', []) + connection_type = module.params.get('connection_type', 'none') + shutdown_all_connections = module.params['shutdown_all_connections'] timeout = module.params['timeout'] interval = module.params['interval'] action = module.params.get('action', 'no_action') @@ -80,20 +123,11 @@ def main(): start_time = time.time() logging.info("start time: %s", datetime.datetime.fromtimestamp(start_time).strftime("%H:%M:%S")) - # interface operation based on action - if action in ["shutdown", "startup"] and shutdown_ports: - ports_str = ",".join(shutdown_ports) - if action == "shutdown": - cmd = "sudo config interface shutdown {}".format(ports_str) - else: - cmd = "sudo config interface startup {}".format(ports_str) - logging.info("The command is: %s", cmd) - rc, out, err = module.run_command(cmd, executable='/bin/bash', use_unsafe_shell=True) - if rc != 0: - module.fail_json(msg=f"Failed to {action} ports: {err}") - logging.info("%s ports: %s", action, ports_str) + if not shutdown_connections or action == 'no_action': + logging.info("No connections or action is 'no_action', skipping interface operation.") else: - logging.info("action is no_action or no shutdown_ports, skip interface operation.") + # interface operation based on action + perform_action(module, action, connection_type, shutdown_connections, shutdown_all_connections) # Sleep some time to wait routes to be converged time.sleep(4) @@ -105,7 +139,7 @@ def main(): logging.info(f"BGP routes check round: {check_count}") # record the time before getting routes in this round before_get_route_time = time.time() - logging.info(f"Before get route time:" + logging.info(f"Before get route time: " f" {datetime.datetime.fromtimestamp(before_get_route_time).strftime('%H:%M:%S')}") running_routes = get_bgp_ipv6_routes(module) logging.info("Obtained the routes") diff --git a/tests/bgp/test_ipv6_bgp_scale.py b/tests/bgp/test_ipv6_bgp_scale.py index cd0f90e5ad6..eaeb3943f21 100644 --- a/tests/bgp/test_ipv6_bgp_scale.py +++ b/tests/bgp/test_ipv6_bgp_scale.py @@ -37,7 +37,8 @@ IPV6_KEY = "ipv6" MAX_BGP_SESSIONS_DOWN_COUNT = 0 MAX_DOWNTIME = 10 # seconds -MAX_DOWNTIME_ONE_PORT_FLAPPING = 30 # seconds +MAX_DOWNTIME_PORT_FLAPPING = 30 # seconds +MAX_BGP_SESSION_DOWNTIME = MAX_DOWNTIME_PORT_FLAPPING # reused MAX_DOWNTIME_UNISOLATION = 300 # seconds MAX_DOWNTIME_NEXTHOP_GROUP_MEMBER_CHANGE = 30 # seconds PKTS_SENDING_TIME_SLOT = 1 # seconds @@ -404,14 +405,18 @@ def remove_routes_with_nexthops(candidate_routes, nexthop_to_remove, result_rout result_routes[prefix] = value -def check_bgp_routes_converged(duthost, expected_routes, shutdown_ports, timeout=MAX_CONVERGENCE_WAIT_TIME, interval=1, +def check_bgp_routes_converged(duthost, expected_routes, shutdown_connections=None, connection_type='none', + shutdown_all_connections=False, timeout=MAX_CONVERGENCE_WAIT_TIME, interval=1, log_path="/tmp", compressed=False, action='no_action'): + shutdown_connections = shutdown_connections or [] logger.info("Start to check bgp routes converged") expected_routes_json = json.dumps(expected_routes, separators=(',', ':')) result = duthost.check_bgp_ipv6_routes_converged( expected_routes=expected_routes_json, - shutdown_ports=shutdown_ports, + shutdown_connections=shutdown_connections, + connection_type=connection_type, + shutdown_all_connections=shutdown_all_connections, timeout=timeout, interval=interval, log_path=log_path, @@ -431,11 +436,21 @@ def check_bgp_routes_converged(duthost, expected_routes, shutdown_ports, timeout } return ret else: - # When routes convergence fail, if the action is shutdown and shutdown_ports is not empty, restore interfaces - if action == 'shutdown' and shutdown_ports: - logger.info(f"Recover interfaces {shutdown_ports} after failure") - duthost.no_shutdown_multiple(shutdown_ports) - pytest.fail(f"BGP routes are not stable in {timeout} seconds") + # When routes convergence fail, if the action is shutdown and shutdown_connections is not empty + # restore interfaces + if action == 'shutdown' and shutdown_connections: + if connection_type == 'ports': + logger.info(f"Recover interfaces {shutdown_connections} after failure") + duthost.no_shutdown_multiple(shutdown_connections) + if connection_type == 'bgp_sessions': + if shutdown_all_connections: + logger.info("Recover all BGP sessions after failure") + duthost.shell("sudo config bgp startup all") + else: + for session in shutdown_connections: + logger.info(f"Recover BGP session {session} after failure") + duthost.shell(f"sudo config bgp startup neighbor {session}") + pytest.fail(f"BGP routes aren't stable in {timeout} seconds") def compress_expected_routes(expected_routes): @@ -445,6 +460,123 @@ def compress_expected_routes(expected_routes): return b64_str +def _select_targets(bgp_peers_info, all_flap, flapping_count): + """Selects flapping_neighbors, injection_neighbor, flapping_ports, injection_port.""" + bgp_neighbors = list(bgp_peers_info.keys()) + pytest_assert(len(bgp_neighbors) >= 2, "At least two BGP neighbors required for flap test") + if all_flap: + flapping_neighbors = bgp_neighbors + injection_neighbor = random.choice(bgp_neighbors) + logger.info(f"[FLAP TEST] All neighbors are flapping: {len(flapping_neighbors)}") + else: + flapping_neighbors = random.sample(bgp_neighbors, flapping_count) + injection_candidates = [n for n in bgp_neighbors if n not in flapping_neighbors] + injection_neighbor = random.choice(injection_candidates) + logger.info(f"[FLAP TEST] Flapping neighbors count: {len(flapping_neighbors)}, " + f"Flapping neighbors: {flapping_neighbors}") + flapping_ports = [bgp_peers_info[n][DUT_PORT] for n in flapping_neighbors] + injection_dut_port = bgp_peers_info[injection_neighbor][DUT_PORT] + injection_port = [info[PTF_PORT] for info in bgp_peers_info.values() if info[DUT_PORT] == injection_dut_port][0] + logger.info(f"Flapping ports: {flapping_ports}") + logger.info(f"[FLAP TEST] Injection neighbor: {injection_neighbor}, Injection DUT port: {injection_dut_port}") + logger.info("Injection port: %s", injection_port) + return flapping_neighbors, injection_neighbor, flapping_ports, injection_port + + +def flapper(duthost, pdp, bgp_peers_info, transient_setup, flapping_count, connection_type, action, downtime_threshold): + """ + Orchestrates interface/BGP session flapping and recovery on the DUT, generating test traffic to assess both + control and data plane convergence behavior. This function is designed for use in test scenarios + where some or all BGP neighbors or ports are shut down and restarted. + + Behavior: + - On shutdown action: Randomly selects (or selects all) BGP neighbors/ports to flap, as well as an injection port + to use for sending traffic during the event. It computes expected post-flap routes and sets up traffic streams. + - On startup action: Reuses the previously determined injection/flapping selections to restore connectivity and + again validates route convergence and traffic recovery. + - Measures and validates data plane downtime across the operations, helping to detect issues in convergence times. + - Returns details about the selected connections and test traffic for subsequent phases. + + Returns: + For shutdown phase: dict with flapping_connections, injection_port, compressed_startup_routes, prefixes. + For startup phase: empty dict. + """ + global global_icmp_type, current_test, test_results + current_test = f"flapper_{action}_{connection_type}_count_{flapping_count}" + global_icmp_type += 1 + exp_mask = setup_packet_mask_counters(pdp, global_icmp_type) + all_flap = (flapping_count == 'all') + + # Currently treating the shutdown action as a setup mechanism for a startup action to follow. + # So we only do the selection of flapping and injection neighbors when action is shutdown + # And we reuse the same selection for startup action + if action == 'shutdown': + bgp_neighbors = list(bgp_peers_info.keys()) + pytest_assert(len(bgp_neighbors) >= 2, "At least two BGP neighbors required for flap test") + + # Choose target neighbors (to flap) and injection (to keep traffic stable) + flapping_neighbors, injection_neighbor, flapping_ports, injection_port = _select_targets( + bgp_peers_info, all_flap, flapping_count + ) + + flapping_connections = {'ports': flapping_ports, 'bgp_sessions': flapping_neighbors}.get(connection_type, []) + # Build expected routes after shutdown + startup_routes = get_all_bgp_ipv6_routes(duthost, save_snapshot=False) + neighbor_ecmp_routes = get_ecmp_routes(startup_routes, bgp_peers_info) + prefixes = neighbor_ecmp_routes[injection_neighbor] + nexthops_to_remove = [b[IPV6_KEY] for b in bgp_peers_info.values() if b[DUT_PORT] in flapping_ports] + expected_routes = deepcopy(startup_routes) + remove_routes_with_nexthops(startup_routes, nexthops_to_remove, expected_routes) + compressed_routes = compress_expected_routes(expected_routes) + else: + compressed_routes = transient_setup['compressed_startup_routes'] + injection_port = transient_setup['injection_port'] + flapping_connections = transient_setup['flapping_connections'] + prefixes = transient_setup['prefixes'] + + pkts = generate_packets( + prefixes, + duthost.facts['router_mac'], + pdp.get_mac(pdp.port_to_device(injection_port), injection_port) + ) + + terminated = Event() + traffic_thread = Thread( + target=send_packets, args=(terminated, pdp, pdp.port_to_device(injection_port), injection_port, pkts) + ) + flush_counters(pdp, exp_mask) + traffic_thread.start() + start_time = datetime.datetime.now() + try: + result = check_bgp_routes_converged( + duthost=duthost, + expected_routes=compressed_routes, + shutdown_connections=flapping_connections, + connection_type=connection_type, + shutdown_all_connections=all_flap, + timeout=MAX_CONVERGENCE_WAIT_TIME, + compressed=True, + action=action + ) + terminated.set() + traffic_thread.join() + end_time = datetime.datetime.now() + validate_rx_tx_counters(pdp, end_time, start_time, exp_mask, downtime_threshold) + if not result.get("converged"): + pytest.fail("BGP routes are not stable in long time") + finally: + # Ensure traffic is stopped + terminated.set() + traffic_thread.join() + + return { + "flapping_connections": flapping_connections, + "injection_port": injection_port, + "compressed_startup_routes": compress_expected_routes(startup_routes), + "prefixes": prefixes + } if action == 'shutdown' else {} + + def test_port_flap_with_syslog( request, duthost, @@ -469,10 +601,12 @@ def test_port_flap_with_syslog( duthost.shell('sudo logger "%s"' % LOG_STAMP) try: result = check_bgp_routes_converged( - duthost, - compressed_expected_routes, - flapping_ports, - MAX_CONVERGENCE_WAIT_TIME, + duthost=duthost, + expected_routes=compressed_expected_routes, + shutdown_connections=flapping_ports, + connection_type='ports', + shutdown_all_connections=False, + timeout=MAX_CONVERGENCE_WAIT_TIME, compressed=True, action='shutdown' ) @@ -503,7 +637,7 @@ def test_port_flap_with_syslog( duthost.no_shutdown_multiple(flapping_ports) -@pytest.mark.parametrize("flapping_port_count", [1, 10, 20]) +@pytest.mark.parametrize("flapping_port_count", [1, 10, 20, 'all']) def test_sessions_flapping( request, duthost, @@ -513,78 +647,53 @@ def test_sessions_flapping( setup_routes_before_test ): ''' - This test is to make sure When BGP sessions are flapping, - control plane is functional and data plane has no downtime or acceptable downtime. - Steps: - Start and keep sending packets with all routes to the random one open port via ptf. - Shutdown flapping_port_count random port(s) that establishing bgp sessions. - Wait for routes are stable, check if all nexthops connecting the shut down ports are disappeared in routes. - Stop packet sending - Estimate data plane down time by check packet count sent, received and duration. + Validates that both control plane and data plane remain functional with acceptable downtime when ports are + flapped (brought down and back up), simulating various failure or maintenance scenarios. + + Uses the flapper function to orchestrate the flapping of ports and measure convergence times. + + Parameters range from flapping a single port to all ports. + Expected result: - Dataplane downtime is less than MAX_DOWNTIME_ONE_PORT_FLAPPING. + Dataplane downtime is less than MAX_DOWNTIME_PORT_FLAPPING or MAX_DOWNTIME_UNISOLATION for all ports. ''' - global current_test - current_test = request.node.name + f"_flapping_port_count_{flapping_port_count}" - global global_icmp_type - global_icmp_type += 1 pdp = ptfadapter.dataplane pdp.set_qlen(PACKET_QUEUE_LENGTH) - exp_mask = setup_packet_mask_counters(pdp, global_icmp_type) - bgp_neighbors = [hostname for hostname in bgp_peers_info.keys()] + downtime_threshold = MAX_DOWNTIME_UNISOLATION if flapping_port_count == 'all' else MAX_DOWNTIME_PORT_FLAPPING + # Measure shutdown convergence + transient_setup = flapper(duthost, pdp, bgp_peers_info, None, flapping_port_count, 'ports', + 'shutdown', downtime_threshold) + # Measure startup convergence + flapper(duthost, pdp, None, transient_setup, flapping_port_count, 'ports', 'startup', downtime_threshold) - # Select flapping ports randomly - random.shuffle(bgp_neighbors) - flapping_neighbors, unflapping_neighbors = bgp_neighbors[:flapping_port_count], bgp_neighbors[flapping_port_count:] - flapping_ports = [bgp_peers_info[neighbor][DUT_PORT] for neighbor in flapping_neighbors] - unflapping_ports = [bgp_peers_info[neighbor][DUT_PORT] for neighbor in unflapping_neighbors] - logger.info("Flapping_port_count is %d, flapping ports: %s and unflapping ports %s", - flapping_port_count, flapping_ports, unflapping_ports) - # Select a random unflapping neighbor to send packets - injection_bgp_neighbor = random.choice(unflapping_neighbors) - injection_dut_port = bgp_peers_info[injection_bgp_neighbor][DUT_PORT] - logger.info("Injection BGP neighbor: %s. Injection dut port: %s", injection_bgp_neighbor, injection_dut_port) - injection_port = [i[PTF_PORT] for i in bgp_peers_info.values() if i[DUT_PORT] == injection_dut_port][0] - logger.info("Injection port: %s", injection_port) +@pytest.mark.parametrize("flapping_neighbor_count", [1, 10]) +def test_bgp_admin_flap( + request, + duthost, + ptfadapter, + bgp_peers_info, + flapping_neighbor_count +): + """ + Validates that both control plane and data plane remain functional with acceptable downtime when BGP sessions are + flapped (brought down and back up), simulating various failure or maintenance scenarios. - startup_routes = get_all_bgp_ipv6_routes(duthost, True) - neighbor_ecmp_routes = get_ecmp_routes(startup_routes, bgp_peers_info) - pkts = generate_packets( - neighbor_ecmp_routes[injection_bgp_neighbor], - duthost.facts['router_mac'], - pdp.get_mac(pdp.port_to_device(injection_port), injection_port) - ) + Uses the flapper function to orchestrate the flapping of BGP sessions and measure convergence times. - nexthops_to_remove = [b[IPV6_KEY] for b in bgp_peers_info.values() if b[DUT_PORT] in flapping_ports] - expected_routes = deepcopy(startup_routes) - remove_routes_with_nexthops(startup_routes, nexthops_to_remove, expected_routes) - compressed_expected_routes = compress_expected_routes(expected_routes) - terminated = Event() - traffic_thread = Thread( - target=send_packets, args=(terminated, pdp, pdp.port_to_device(injection_port), injection_port, pkts) - ) - flush_counters(pdp, exp_mask) - traffic_thread.start() - start_time = datetime.datetime.now() + Parameters range from flapping a single session to all sessions. - try: - result = check_bgp_routes_converged( - duthost, - compressed_expected_routes, - flapping_ports, - MAX_CONVERGENCE_WAIT_TIME, - compressed=True, - action='shutdown' - ) - terminated.set() - traffic_thread.join() - end_time = datetime.datetime.now() - validate_rx_tx_counters(pdp, end_time, start_time, exp_mask, MAX_DOWNTIME_ONE_PORT_FLAPPING) - if not result.get("converged"): - pytest.fail("BGP routes are not stable in long time") - finally: - duthost.no_shutdown_multiple(flapping_ports) + Expected result: + Dataplane downtime is less than MAX_BGP_SESSION_DOWNTIME or MAX_DOWNTIME_UNISOLATION for all ports. + """ + pdp = ptfadapter.dataplane + pdp.set_qlen(PACKET_QUEUE_LENGTH) + downtime_threshold = MAX_DOWNTIME_UNISOLATION if flapping_neighbor_count == 'all' else MAX_BGP_SESSION_DOWNTIME + # Measure shutdown convergence + transient_setup = flapper(duthost, pdp, bgp_peers_info, None, flapping_neighbor_count, + 'bgp_sessions', 'shutdown', downtime_threshold) + # Measure startup convergence + flapper(duthost, pdp, None, transient_setup, flapping_neighbor_count, 'bgp_sessions', 'startup', downtime_threshold) def test_nexthop_group_member_scale( @@ -678,10 +787,12 @@ def test_nexthop_group_member_scale( try: compressed_expected_routes = compress_expected_routes(expected_routes) result = check_bgp_routes_converged( - duthost, - compressed_expected_routes, - [], - MAX_CONVERGENCE_WAIT_TIME, + duthost=duthost, + expected_routes=compressed_expected_routes, + shutdown_connections=[], + connection_type='none', + shutdown_all_connections=False, + timeout=MAX_CONVERGENCE_WAIT_TIME, compressed=True, action='no_action' ) @@ -729,10 +840,12 @@ def test_nexthop_group_member_scale( servers_dut_interfaces.get(ptf_ip, '')) compressed_startup_routes = compress_expected_routes(startup_routes) result = check_bgp_routes_converged( - duthost, - compressed_startup_routes, - [], - MAX_CONVERGENCE_WAIT_TIME, + duthost=duthost, + expected_routes=compressed_startup_routes, + shutdown_connections=[], + connection_type='none', + shutdown_all_connections=False, + timeout=MAX_CONVERGENCE_WAIT_TIME, compressed=True, action='no_action' ) @@ -742,89 +855,3 @@ def test_nexthop_group_member_scale( validate_rx_tx_counters(pdp, end_time, start_time, exp_mask, MAX_DOWNTIME_NEXTHOP_GROUP_MEMBER_CHANGE) if not result.get("converged"): pytest.fail("BGP routes are not stable in long time") - - -def test_device_unisolation( - request, - duthost, - ptfadapter, - bgp_peers_info, - setup_routes_before_test, - tbinfo -): - ''' - This test is for the worst scenario that all ports are flapped, - verify control/data plane have acceptable convergence time. - Steps: - Shut down all ports on device. (shut down T1 sessions ports on T0 DUT, shut down T0 sessions ports on T1 DUT.) - Wait for routes are stable. - Start and keep sending packets with all routes to all ports via ptf. - Startup all ports and wait for routes are stable. - Stop sending packets. - Estimate control/data plane convergence time. - Expected result: - Dataplane downtime is less than MAX_DOWNTIME_UNISOLATION. - ''' - global current_test - current_test = request.node.name - global global_icmp_type - global_icmp_type += 1 - pdp = ptfadapter.dataplane - pdp.set_qlen(PACKET_QUEUE_LENGTH) - exp_mask = setup_packet_mask_counters(pdp, global_icmp_type) - - bgp_ports = [bgp_info[DUT_PORT] for bgp_info in bgp_peers_info.values()] - - injection_bgp_neighbor = random.choice(list(bgp_peers_info.keys())) - injection_dut_port = bgp_peers_info[injection_bgp_neighbor][DUT_PORT] - injection_port = [i[PTF_PORT] for i in bgp_peers_info.values() if i[DUT_PORT] == injection_dut_port][0] - logger.info("Injection port: %s", injection_port) - - startup_routes = get_all_bgp_ipv6_routes(duthost, True) - neighbor_ecmp_routes = get_ecmp_routes(startup_routes, bgp_peers_info) - pkts = generate_packets( - neighbor_ecmp_routes[injection_bgp_neighbor], - duthost.facts['router_mac'], - pdp.get_mac(pdp.port_to_device(injection_port), injection_port) - ) - - nexthops_to_remove = [b[IPV6_KEY] for b in bgp_peers_info.values() if b[DUT_PORT] in bgp_ports] - expected_routes = deepcopy(startup_routes) - remove_routes_with_nexthops(startup_routes, nexthops_to_remove, expected_routes) - try: - compressed_expected_routes = compress_expected_routes(expected_routes) - result = check_bgp_routes_converged( - duthost, - compressed_expected_routes, - bgp_ports, - MAX_CONVERGENCE_WAIT_TIME, - compressed=True, - action='shutdown' - ) - if not result.get("converged"): - pytest.fail("BGP routes are not stable in long time") - except Exception: - duthost.no_shutdown_multiple(bgp_ports) - - terminated = Event() - traffic_thread = Thread( - target=send_packets, args=(terminated, pdp, pdp.port_to_device(injection_port), injection_port, pkts) - ) - flush_counters(pdp, exp_mask) - start_time = datetime.datetime.now() - traffic_thread.start() - compressed_expected_routes = compress_expected_routes(startup_routes) - result = check_bgp_routes_converged( - duthost, - compressed_expected_routes, - bgp_ports, - MAX_CONVERGENCE_WAIT_TIME, - compressed=True, - action='startup' - ) - terminated.set() - traffic_thread.join() - end_time = datetime.datetime.now() - validate_rx_tx_counters(pdp, end_time, start_time, exp_mask, MAX_DOWNTIME_UNISOLATION) - if not result.get("converged"): - pytest.fail("BGP routes are not stable in long time")