Dual tor failure utilities and sample tests

tests/common/dualtor/mux_simulator_control.py

@@ -196,6 +196,21 @@ def check_simulator_read_side(mux_server_url, physical_port):
     else:
         return -1
 
+@pytest.fixture
+def get_active_torhost(mux_server_url, upper_tor_host, lower_tor_host):
+
+    def get_active_torhost():
+        active_tor_host = None
+        active_side = check_simulator_read_side(mux_server_url, 1)
+        pytest_assert(active_side != -1, "Failed to retrieve the current active tor from y_cable simulator server")
+        if active_side == 1:
+            active_tor_host = upper_tor_host
+        elif active_side == 2:
+            active_tor_host = lower_tor_host
+        return active_tor_host
+
+    return get_active_torhost
+
 @pytest.fixture(scope='module')
 def toggle_all_simulator_ports_to_tor_a(mux_server_url):
     """

tests/common/dualtor/tor_failure_utils.py

@@ -0,0 +1,124 @@
+"""
+Tor Failure utilities to test switchover and MUX handling during:
+Shutdown all BGP sessions on a ToR
+Shutdown the LinkProber on a ToR
+Blackhole all traffic on a ToR
+Reboot a ToR
+"""
+from tests.common.reboot import *
+import ipaddress
+import pytest
+import logging
+import time
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def shutdown_tor_bgp():
+    """
+    Shutdown all BGP sessions 
+    """
+    torhost = []
+    def shutdown_tor_bgp(duthost, shutdown_all=True):
+        torhost.append(duthost)
+        bgp_neighbors = duthost.get_bgp_neighbors()
+        up_bgp_neighbors = [ k.lower() for k, v in bgp_neighbors.items() if v["state"] == "established" ]
+        if shutdown_all and up_bgp_neighbors:
+            logger.info("Shutdown BGP sessions on {}".format(duthost.hostname))
+            duthost.shell("config bgp shutdown all")
+
+    yield shutdown_tor_bgp
+
+    time.sleep(1)
+    if torhost:
+        duthost = torhost[0]
+        logger.info("Starting BGP sessions on {}".format(duthost.hostname))
+        duthost.shell("config bgp startup all")
+
+
+@pytest.fixture
+def shutdown_tor_heartbeat():
+    """
+    Shutdown the LinkProber
+    """
+    torhost = []
+    def shutdown_tor_heartbeat(duthost):
+        # TODO - verify support after LinkProber submodule is ready
+        torhost.append(duthost)
+        duthost.shell("sudo systemctl stop mux")
+        duthost.shell("sudo systemctl disable mux")
+
+    yield shutdown_tor_heartbeat
+
+    if torhost:
+        duthost = torhost[0]
+        duthost.shell("sudo systemctl start mux")
+        duthost.shell("sudo systemctl enable mux")
+
+
+@pytest.fixture
+def tor_blackhole_traffic():
+    """
+    Configure tor to blackhole all traffic
+    Install a blackhole route
+    """
+    torhost = []
+    def tor_blackhole_traffic(duthost, kernel=False, asic=False):
+        torhost.append(duthost)
+        if asic:
+            duthost.shell("ip route del 0.0.0.0/0")
+        elif kernel:
+            pass # TODO
+
+    yield tor_blackhole_traffic
+
+    if torhost:
+        duthost = torhost[0]
+        lo_ipv4 = None
+        lo_ipv6 = None
+        config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts']
+        los = config_facts.get("LOOPBACK_INTERFACE", {})
+        logger.info("Loopback IPs: {}".format(los))
+        for k, v in los.items():
+            if k == "Loopback0":
+                for ipstr in v.keys():
+                    ip = ipaddress.ip_interface(ipstr)
+                    if ip.version == 4:
+                        lo_ipv4 = ip
+                    elif ip.version == 6:
+                        lo_ipv6 = ip
+
+        duthost.shell("ip -4 route add 0.0.0.0/0 nexthop via {}".format(lo_ipv4.ip))
+        #duthost.shell("ip -6 route add ::/0 nexthop via {}".format(lo_ipv6.ip))
+
+@pytest.fixture
+def reboot_tor(localhost):
+    """
+    Reboot TOR
+    """
+    torhost = []
+    torhost_ip = []
+    def reboot_tor(duthost, reboot_type=REBOOT_TYPE_COLD):
+        torhost.append(duthost)
+        torhost_ip.append(duthost.setup()['ansible_facts']['ansible_eth0']['ipv4']['address'])
+        logger.info("Shutdown BGP sessions on {}".format(duthost.hostname))
+        reboot(duthost, localhost, reboot_type=reboot_type, wait_for_ssh=False)
+
+    yield reboot_tor
+    # TODO Add IO check capability
+
+    if torhost:
+        duthost = torhost[0]
+        dut_ip = torhost_ip[0]
+        logger.info("Waiting for ssh to startup on {}".format((duthost.hostname)))
+        res = localhost.wait_for(host=dut_ip,
+                                port=SONIC_SSH_PORT,
+                                state='started',
+                                search_regex=SONIC_SSH_REGEX,
+                                delay=10,
+                                timeout=300,
+                                module_ignore_errors=True)
+        if res.is_failed or ('msg' in res and 'Timeout' in res['msg']):
+            raise Exception("DUT {} did not startup after reboot".format((duthost.hostname)))
+        logger.info("SSH started on {}".format((duthost.hostname)))

tests/common/reboot.py

@@ -75,7 +75,8 @@ def get_warmboot_finalizer_state(duthost):
         finalizer_state = err.results
     return finalizer_state
 
-def reboot(duthost, localhost, reboot_type='cold', delay=10, timeout=0, wait=0, reboot_helper=None, reboot_kwargs=None):
+def reboot(duthost, localhost, reboot_type='cold', delay=10, \
+    timeout=0, wait=0, wait_for_ssh=True, reboot_helper=None, reboot_kwargs=None):
     """
     reboots DUT
     :param duthost: DUT host object
@@ -134,6 +135,9 @@ def execute_reboot_helper():
             logger.error('reboot result: {}'.format(reboot_res.get()))
         raise Exception('DUT did not shutdown')
 
+    if not wait_for_ssh:
+        return
+
     # TODO: add serial output during reboot for better debuggability
     #       This feature requires serial information to be present in
     #       testbed information

tests/dualtor/tor_failure/test_tor_component_failure.py

@@ -0,0 +1,59 @@
+import pytest
+from tests.common.dualtor.tor_failure_utils import shutdown_tor_bgp
+from tests.common.dualtor.dual_tor_utils import upper_tor_host, lower_tor_host # lgtm[py/unused-import]
+from tests.common.dualtor.mux_simulator_control import get_active_torhost
+from tests.common.dualtor.mux_simulator_control import mux_server_url # lgtm[py/unused-import]
+from tests.common.helpers.assertions import pytest_assert
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def test_active_tor_bgp_down_upstream(shutdown_tor_bgp, get_active_torhost):
+    active_host_before = get_active_torhost()
+    logger.info("Active ToR before shutting down bgp sessions {}".format(active_host_before.hostname))
+
+    # perform bgp shutdown on active TOR
+    shutdown_tor_bgp(active_host_before)
+    # TODO add more verification steps here
+
+    active_host_after = get_active_torhost()
+    logger.info("Active ToR after shutting down bgp sessions {}".format(active_host_after.hostname))
+
+    pytest_assert(active_host_before != active_host_after, "ToR switchover failed")
+
+def test_active_tor_bgp_down_downstream_active(): # Out of scope, method included for completeness
+    pass
+
+def test_active_tor_bgp_down_downstream_standby():
+    pass
+
+
+def test_standby_tor_bgp_down_upstream():
+    pass
+
+def test_standby_tor_bgp_down_downstream_active():
+    pass
+
+def test_standby_tor_bgp_down_downstream_standby(): # Out of scope, method included for completeness
+    pass
+
+
+def test_active_tor_heartbeat_loss_upstream():
+    pass
+
+def test_active_tor_heartbeat_loss_downstream_active():
+    pass
+
+def test_active_tor_heartbeat_loss_downstream_standby():
+    pass
+
+
+def test_standby_tor_heartbeat_loss_upstream():
+    pass
+
+def test_standby_tor_heartbeat_loss_downstream_active():
+    pass
+
+def test_standby_tor_heartbeat_loss_downstream_standby():
+    pass

tests/dualtor/tor_failure/test_tor_failure.py

@@ -0,0 +1,79 @@
+import pytest
+from tests.common.dualtor.tor_failure_utils import tor_blackhole_traffic, reboot_tor
+from tests.common.dualtor.dual_tor_utils import upper_tor_host, lower_tor_host # lgtm[py/unused-import]
+from tests.common.dualtor.mux_simulator_control import get_active_torhost
+from tests.common.dualtor.mux_simulator_control import mux_server_url # lgtm[py/unused-import]
+from tests.common.helpers.assertions import pytest_assert
+import logging
+
+logger = logging.getLogger(__name__)
+
+def test_active_tor_failure_upstream(tor_blackhole_traffic, get_active_torhost):
+    active_host_before = get_active_torhost()
+    logger.info("Active ToR before traffic blackhole {}".format(active_host_before.hostname))
+
+    # Blackhole all the traffic on active tor on ASIC
+    tor_blackhole_traffic(active_host_before, asic=True)
+    # TODO - add other checks and IO verification here
+
+    active_host_after = get_active_torhost()
+    logger.info("Active ToR after traffic blackhole {}".format(active_host_after.hostname))
+
+    pytest_assert(active_host_before != active_host_after, "ToR switchover failed")
+
+
+def test_active_tor_failure_downstream_active():
+    # Out of scope, method included for completeness
+    pass
+
+
+def test_active_tor_failure_downstream_standby():
+    pass
+
+
+def test_standby_tor_failure_upstream():
+    pass
+
+
+def test_standby_tor_failure_downstream_active():
+    pass
+
+
+def test_standby_tor_failure_downstream_standby():
+    # Out of scope, method included for completeness
+    pass
+
+def test_active_tor_reboot_upstream(reboot_tor, get_active_torhost):
+    active_host_before = get_active_torhost()
+    logger.info("Active ToR before performing reboot {}".format(active_host_before.hostname))
+
+    # perform reboot on the active tor
+    reboot_tor(active_host_before)
+    # TODO - add other checks and IO verification here
+
+    active_host_after = get_active_torhost()
+    logger.info("Active ToR after performing reboot {}".format(active_host_after.hostname))
+
+    pytest_assert(active_host_before != active_host_after, "ToR switchover failed")
+
+
+def test_active_tor_reboot_downstream_active():
+    # Out of scope, method included for completeness
+    pass
+
+
+def test_active_tor_reboot_downstream_standby():
+    pass
+
+
+def test_standby_tor_reboot_upstream():
+    pass
+
+
+def test_standby_tor_reboot_downstream_active():
+    pass
+
+
+def test_standby_tor_reboot_downstream_standby():
+    # Out of scope, method included for completeness
+    pass