Re-enable ICMP responder and gratuitous ARP service for active-standby dualtor topologies.

tests/arp/test_unknown_mac.py

@@ -14,9 +14,11 @@
 from tests.common import constants
 from tests.common.fixtures.ptfhost_utils import change_mac_addresses    # noqa F401
 from tests.common.fixtures.ptfhost_utils import copy_arp_responder_py   # noqa F401
+from tests.common.fixtures.ptfhost_utils import pause_garp_service_dualtor_m      # noqa F401
+from tests.common.fixtures.ptfhost_utils import pause_icmp_responder_dualtor_m    # noqa F401
 from tests.common.helpers.assertions import pytest_assert, pytest_require
 from tests.common.dualtor.dual_tor_utils import mux_cable_server_ip
-from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_m    # noqa F401
+from tests.common.dualtor.mux_simulator_control import toggle_all_simulator_ports_to_rand_selected_tor_unconditionally    # noqa F401
 from tests.common.utilities import get_intf_by_sub_intf, wait_until
 
 logger = logging.getLogger(__name__)
@@ -42,7 +44,7 @@ def wrapper(self, *args):
 
 
 @pytest.fixture(autouse=True, scope="module")
-def unknownMacSetup(duthosts, rand_one_dut_hostname, tbinfo):
+def unknownMacSetup(duthosts, rand_one_dut_hostname, tbinfo, pause_icmp_responder_dualtor_m, pause_garp_service_dualtor_m):   # noqa F811
     """
     Fixture to populate all the parameters needed for the test
 
@@ -152,7 +154,7 @@ def flushArpFdb(duthosts, rand_one_dut_hostname):
 
 @pytest.fixture(autouse=True)
 def populateArp(unknownMacSetup, flushArpFdb, ptfhost, duthosts, rand_one_dut_hostname,
-                toggle_all_simulator_ports_to_rand_selected_tor_m):     # noqa F811
+                toggle_all_simulator_ports_to_rand_selected_tor_unconditionally):     # noqa F811
     """
     Fixture to populate ARP entry on the DUT for the traffic destination
 

tests/common/dualtor/mux_simulator_control.py

@@ -11,6 +11,7 @@
 from requests import Session
 
 from tests.common import utilities
+from tests.common.helpers.assertions import pytest_assert as pt_assert
 from tests.common.dualtor.dual_tor_common import cable_type                             # noqa F401
 from tests.common.dualtor.dual_tor_common import mux_config                             # noqa F401
 from tests.common.dualtor.dual_tor_common import CableType
@@ -42,6 +43,9 @@
     'check_mux_status',
     'validate_check_result',
     'simulator_flap_counter',
+    'config_active_standby_ports',
+    'toggle_all_simulator_ports_to_rand_selected_tor_unconditionally',
+    'toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_unconditionally',
     ]
 
 logger = logging.getLogger(__name__)
@@ -953,3 +957,99 @@ def validate_check_result(check_result, duthosts, get_mux_status):
         else:
             logger.error('Failed to get mux status from mux simulator')
         pytest.fail('Toggle mux from simulator test failed')
+
+
+def show_muxcable_status(duthost):
+    """
+    Show muxcable status and parse into a dict
+    """
+    command = "show muxcable status --json"
+    output = json.loads(duthost.shell(command)["stdout"])
+
+    ret = {}
+    for port, muxcable in list(output['MUX_CABLE'].items()):
+        ret[port] = {'status': muxcable['STATUS'], 'health': muxcable['HEALTH']}
+
+    return ret
+
+
+@pytest.fixture
+def config_active_standby_ports(duthosts, active_standby_ports, tbinfo):
+
+    def check_active_standby_port_status(duthost, ports, status):
+        logging.debug("Check mux status for ports {} is {}".format(ports, status))
+        show_mux_status_ret = show_muxcable_status(duthost)
+        logging.debug("show_mux_status_ret: {}".format(json.dumps(show_mux_status_ret, indent=4)))
+        for port in ports:
+            if port not in show_mux_status_ret:
+                return False
+            elif show_mux_status_ret[port]['status'] != status:
+                return False
+        return True
+
+    def _config_the_active_standby_dualtor(active_tor, standby_tor, ports):
+        active_side_commands = []
+        standby_side_commands = []
+        logging.info("Configuring {} as active".format(active_tor.hostname))
+        logging.info("Configuring {} as standby".format(standby_tor.hostname))
+        for port in ports:
+            if port not in active_standby_ports:
+                raise ValueError("Port {} is not in the active-active ports".format(port))
+            active_side_commands.append("config mux mode active {}".format(port))
+            standby_side_commands.append("config mux mode standby {}".format(port))
+
+        active_tor.shell_cmds(cmds=active_side_commands)
+        standby_tor.shell_cmds(cmds=standby_side_commands)
+
+        pt_assert(utilities.wait_until(30, 5, 0, check_active_standby_port_status, active_tor, ports, 'active'),
+                  "Could not config ports {} to active on {}".format(ports, active_tor.hostname))
+        pt_assert(utilities.wait_until(30, 5, 0, check_active_standby_port_status, standby_tor, ports, 'standby'),
+                  "Could not config ports {} to standby on {}".format(ports, standby_tor.hostname))
+
+        ports_to_restore.extend(ports)
+
+    ports_to_restore = []
+
+    yield _config_the_active_standby_dualtor
+
+    if ports_to_restore:
+        restore_cmds = []
+        for port in ports_to_restore:
+            restore_cmds.append("config mux mode auto {}".format(port))
+
+        for duthost in duthosts:
+            duthost.shell_cmds(cmds=restore_cmds)
+
+
+@pytest.fixture
+def toggle_all_simulator_ports_to_rand_selected_tor_unconditionally(
+        active_standby_ports,
+        rand_selected_dut,
+        rand_unselected_dut,
+        tbinfo,
+        config_active_standby_ports):
+
+    if 'dualtor' not in tbinfo['topo']['name'] or not active_standby_ports:
+        logger.info('Skipping toggle on non-dualtor testbed or active-active dualtor topo.')
+        return
+
+    config_active_standby_ports(rand_selected_dut, rand_unselected_dut, active_standby_ports)
+
+
+@pytest.fixture
+def toggle_all_simulator_ports_to_enum_rand_one_per_hwsku_frontend_host_unconditionally(
+        active_standby_ports,
+        enum_rand_one_per_hwsku_frontend_hostname,
+        tbinfo,
+        config_active_standby_ports,
+        upper_tor_host,
+        lower_tor_host,
+        duthosts):
+
+    if 'dualtor' not in tbinfo['topo']['name'] or not active_standby_ports:
+        logger.info('Skipping toggle on non-dualtor testbed or active-active dualtor topo.')
+        return
+    active_tor = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
+    standby_tor = upper_tor_host if active_tor == lower_tor_host else lower_tor_host
+
+    config_active_standby_ports(active_tor, standby_tor, active_standby_ports)

tests/common/fixtures/ptfhost_utils.py

@@ -5,6 +5,7 @@
 import yaml
 import six
 import requests
+import re
 
 from ipaddress import ip_interface
 from jinja2 import Template
@@ -235,7 +236,7 @@ def ptf_portmap_file_module(rand_selected_dut, ptfhost, tbinfo):
 def run_icmp_responder_session(duthosts, duthost, ptfhost, tbinfo):
     """Run icmp_responder on ptfhost session-wise on dualtor testbeds with active-active ports."""
     # No vlan is available on non-t0 testbed, so skip this fixture
-    if "dualtor-mixed" not in tbinfo["topo"]["name"] and "dualtor-aa" not in tbinfo["topo"]["name"]:
+    if "dualtor" not in tbinfo["topo"]["name"]:
         logger.info("Skip running icmp_responder at session level, "
                     "it is only for dualtor testbed with active-active mux ports.")
         yield
@@ -269,7 +270,7 @@ def run_icmp_responder_session(duthosts, duthost, ptfhost, tbinfo):
 
     yield
 
-    logger.info("Leave icmp_responder running for dualtor-mixed/dualtor-aa topology")
+    logger.info("Leave icmp_responder running for dualtor/dualtor-aa/dualtor-mixed topology")
     return
 
 
@@ -348,8 +349,7 @@ def pause_garp_service(ptfhost):
         ptfhost.shell("supervisorctl start garp_service")
 
 
-@pytest.fixture(scope='module', autouse=True)
-def run_garp_service(duthost, ptfhost, tbinfo, change_mac_addresses, request):
+def _run_garp_service(duthost, ptfhost, tbinfo, request):
     config_facts = duthost.config_facts(host=duthost.hostname, source="running")['ansible_facts']
     if tbinfo['topo']['type'] == 't0':
         garp_config = {}
@@ -379,7 +379,6 @@ def run_garp_service(duthost, ptfhost, tbinfo, change_mac_addresses, request):
         if 'dualtor' not in tbinfo['topo']['name']:
             if "test_advanced_reboot" in request.node.name:
                 logger.info("Skip GARP service for advanced-reboot test on non dualtor devices")
-                yield
                 return
             # For mocked dualtor testbed
             mux_cable_table = {}
@@ -423,13 +422,143 @@ def run_garp_service(duthost, ptfhost, tbinfo, change_mac_addresses, request):
     else:
         logger.info("Not running on a T0 testbed, not starting GARP service")
 
+
+@pytest.fixture(scope='module', autouse=True)
+def run_garp_service(duthost, ptfhost, tbinfo, change_mac_addresses, request):
+
+    res = ptfhost.shell("supervisorctl status garp_service", module_ignore_errors=True)
+    if 'RUNNING' in res['stdout']:
+        logger.info("GARP service already running in PTF")
+        yield
+        return
+
+    _run_garp_service(duthost, ptfhost, tbinfo, request)
+
     yield
 
+    if 'dualtor' not in tbinfo['topo']['name'] and "test_advanced_reboot" in request.node.name:
+        return
+
     if tbinfo['topo']['type'] == 't0':
         logger.info("Stopping GARP service on PTF host")
         ptfhost.shell('supervisorctl stop garp_service')
 
 
+def is_active_standby_dualtor_topo(tbinfo):
+    """
+    Check for topologies having all ports in active-standby mode like dualtor. dualtor-120,
+    dualtor-56, etc
+    """
+    topo_name = tbinfo['topo']['name']
+    return re.fullmatch(r"dualtor|dualtor-[0-9]*", topo_name) is not None
+
+
+@pytest.fixture(scope='session', autouse=True)
+def run_garp_service_dualtor_session(duthost, ptfhost, tbinfo, change_mac_addresses, request):
+    if is_active_standby_dualtor_topo(tbinfo):
+        _run_garp_service(duthost, ptfhost, tbinfo, request)
+    yield
+
+
+@pytest.fixture
+def pause_garp_service_dualtor(ptfhost, tbinfo):
+    """
+    Temporarily pause GARP service on PTF for one test method for active-standby dualtor
+
+    `run_garp_service_dualtor_session` is session scoped and autoused,
+    but some tests in modules where it is imported need it disabled
+    This fixture should only be used when garp_service is already running on the PTF
+    """
+    if not is_active_standby_dualtor_topo(tbinfo):
+        yield
+        return
+
+    needs_resume = False
+    res = ptfhost.shell("supervisorctl status garp_service", module_ignore_errors=True)
+    if res['rc'] != 0:
+        logger.warning("GARP service not present on PTF")
+    elif 'RUNNING' in res['stdout']:
+        needs_resume = True
+        ptfhost.shell("supervisorctl stop garp_service")
+    else:
+        logger.warning("GARP service already stopped on PTF")
+
+    yield
+
+    if needs_resume:
+        ptfhost.shell("supervisorctl start garp_service")
+
+
+@pytest.fixture(scope="module")
+def pause_garp_service_dualtor_m(ptfhost, tbinfo):
+    """
+    Temporarily pause GARP service on PTF for one test method for active-standby dualtor
+
+    `run_garp_service_dualtor_session` is session scoped and autoused,
+    but some tests in modules where it is imported need it disabled
+    This fixture should only be used when icmp_responder is already running on the PTF
+    """
+    if not is_active_standby_dualtor_topo(tbinfo):
+        yield
+        return
+
+    needs_resume = False
+    res = ptfhost.shell("supervisorctl status garp_service", module_ignore_errors=True)
+    if res['rc'] != 0:
+        logger.warning("GARP service not present on PTF")
+    elif 'RUNNING' in res['stdout']:
+        needs_resume = True
+        ptfhost.shell("supervisorctl stop garp_service")
+    else:
+        logger.warning("GARP service already stopped on PTF")
+
+    yield
+
+    if needs_resume:
+        ptfhost.shell("supervisorctl start garp_service")
+
+
+@pytest.fixture(scope="module")
+def pause_icmp_responder_dualtor_m(ptfhost, tbinfo):
+    """
+    Temporarily pause ICMP responder service on PTF for one test method for active-standby dualtor
+
+    `run_icmp_responder_session` is session scoped and autoused,
+    but some tests in modules where it is imported need it disabled
+    This fixture should only be used when icmp_responder is already running on the PTF
+    """
+    if not is_active_standby_dualtor_topo(tbinfo):
+        yield
+        return
+
+    icmp_responder_status = ptfhost.shell("supervisorctl status icmp_responder", module_ignore_errors=True)["stdout"]
+    if "RUNNING" not in icmp_responder_status:
+        yield
+        return
+    ptfhost.shell("supervisorctl stop icmp_responder", module_ignore_errors=True)
+
+    yield
+
+    ptfhost.shell("supervisorctl restart icmp_responder", module_ignore_errors=True)
+
+
+@pytest.fixture
+def pause_icmp_responder_dualtor(ptfhost, tbinfo):
+    if not is_active_standby_dualtor_topo(tbinfo):
+        yield
+        return
+
+    icmp_responder_status = ptfhost.shell("supervisorctl status icmp_responder", module_ignore_errors=True)["stdout"]
+    if "RUNNING" not in icmp_responder_status:
+        yield
+        return
+    ptfhost.shell("supervisorctl stop icmp_responder", module_ignore_errors=True)
+
+    yield
+
+    ptfhost.shell("supervisorctl restart icmp_responder", module_ignore_errors=True)
+
+
 def ptf_test_port_map(ptfhost, tbinfo, duthosts, mux_server_url, duts_running_config_facts, duts_minigraph_facts):
     active_dut_map = {}
     if 'dualtor' in tbinfo['topo']['name']:

tests/conftest.py

@@ -61,6 +61,9 @@
 from tests.platform_tests.args.normal_reboot_args import add_normal_reboot_args
 from ptf import testutils
 from ptf.mask import Mask
+from tests.common.fixtures.ptfhost_utils import run_garp_service_dualtor_session                        # noqa F401
+from tests.common.fixtures.ptfhost_utils import change_mac_addresses                    # noqa F401
+
 
 logger = logging.getLogger(__name__)
 cache = FactsCache()