multi-asic support for copp testcase (#3060)

This is PR continuation of #2627 where infra changes were done to support copp on multi-asic platforms. In this PR test is enhanced to work on multi-asic platforms. Following are major changes:

Multi-asic as of now will support installing of nanomsg and ptf_nn_agent in syncd directly mode. This way test case will randomly select any front-asic syncd for verification. To support this needed to add iptable rules for HTTP and PTF Traffic forwarding from host to asic namespace. swap_syncd mode is not supported as of now.

All the docker name are translated to particular namespace/asic that is under test. (Added utility api for same).

Remove disable_lldp_for_testing fixture as noted below since test case is not dependent on that.

Author: abdosi

tests/common/devices/multi_asic.py

@@ -129,6 +129,15 @@ def asic_instance(self, asic_index):
             return self.asics[0]
         return self.asics[asic_index]
 
+    def asic_instance_from_namespace(self, namespace=DEFAULT_NAMESPACE):
+        if not namespace:
+            return self.asics[0]
+
+        for asic in self.asics:
+            if asic.namespace == namespace:
+                return asic
+        return None
+
     def get_asic_ids(self):
         if self.sonichost.facts['num_asic'] == 1:
             return [DEFAULT_ASIC_ID]

tests/copp/copp_utils.py

@@ -4,6 +4,7 @@
     Todo:
         Refactor ptfadapter so it can be leveraged in these test cases.
 """
+import re
 
 DEFAULT_NN_TARGET_PORT = 3
 
@@ -12,20 +13,21 @@
 _UPDATE_COPP_SCRIPT = "copp/scripts/update_copp_config.py"
 
 _BASE_COPP_CONFIG = "/tmp/base_copp_config.json"
-_APP_DB_COPP_CONFIG = "swss:/etc/swss/config.d/00-copp.config.json"
+_APP_DB_COPP_CONFIG = ":/etc/swss/config.d/00-copp.config.json"
 _CONFIG_DB_COPP_CONFIG = "/etc/sonic/copp_cfg.json"
 _TEMP_COPP_CONFIG = "/tmp/copp_config.json"
 _TEMP_COPP_TEMPLATE = "/tmp/copp.json.j2"
 _COPP_TEMPLATE_PATH = "/usr/share/sonic/templates/copp.json.j2"
-_SWSS_COPP_TEMPLATE = "swss:" + _COPP_TEMPLATE_PATH
+_SWSS_COPP_TEMPLATE = ":" + _COPP_TEMPLATE_PATH
 
 _PTF_NN_TEMPLATE = "templates/ptf_nn_agent.conf.ptf.j2"
 _PTF_NN_DEST = "/etc/supervisor/conf.d/ptf_nn_agent.conf"
 
 _SYNCD_NN_TEMPLATE = "templates/ptf_nn_agent.conf.dut.j2"
 _SYNCD_NN_DEST = "/tmp/ptf_nn_agent.conf"
+_SYNCD_NN_FILE = "ptf_nn_agent.conf"
 
-def limit_policer(dut, pps_limit):
+def limit_policer(dut, pps_limit, nn_target_namespace):
     """
         Updates the COPP configuration in the SWSS container to respect a given rate limit.
 
@@ -37,8 +39,12 @@ def limit_policer(dut, pps_limit):
             pps_limit (int): The rate limit for COPP to enforce on ALL trap groups.
     """
 
+    asichost = dut.asic_instance_from_namespace(nn_target_namespace)
+
+    swss_docker_name = asichost.get_docker_name("swss")
+
     if "201811" in dut.os_version or "201911" in dut.os_version:
-        dut.command("docker cp {} {}".format(_APP_DB_COPP_CONFIG, _BASE_COPP_CONFIG))
+        dut.command("docker cp {} {}".format(swss_docker_name + _APP_DB_COPP_CONFIG, _BASE_COPP_CONFIG))
         config_format = "app_db"
     else:
         dut.command("cp {} {}".format(_CONFIG_DB_COPP_CONFIG, _BASE_COPP_CONFIG))
@@ -53,17 +59,17 @@ def limit_policer(dut, pps_limit):
     )
 
     if config_format == "app_db":
-        dut.command("docker cp {} {}".format(_TEMP_COPP_CONFIG, _APP_DB_COPP_CONFIG))
+        dut.command("docker cp {} {}".format(_TEMP_COPP_CONFIG, swss_docker_name + _APP_DB_COPP_CONFIG))
 
         # As copp config is regenerated each time swss starts need to replace the template with
         # config updated above. But before doing that need store the original template in a
         # temporary file for restore after test.
-        dut.command("docker cp {} {}".format(_SWSS_COPP_TEMPLATE, _TEMP_COPP_TEMPLATE))
-        dut.command("docker cp {} {}".format(_TEMP_COPP_CONFIG, _SWSS_COPP_TEMPLATE))
+        dut.command("docker cp {} {}".format(swss_docker_name + _SWSS_COPP_TEMPLATE, _TEMP_COPP_TEMPLATE))
+        dut.command("docker cp {} {}".format(_TEMP_COPP_CONFIG, swss_docker_name + _SWSS_COPP_TEMPLATE))
     else:
         dut.command("cp {} {}".format(_TEMP_COPP_CONFIG, _CONFIG_DB_COPP_CONFIG))
 
-def restore_policer(dut):
+def restore_policer(dut, nn_target_namespace):
     """
         Reloads the default COPP configuration in the SWSS container.
 
@@ -72,10 +78,14 @@ def restore_policer(dut):
 
             The SWSS container must be restarted for the config change to take effect.
     """
+    asichost = dut.asic_instance_from_namespace(nn_target_namespace)
+
+    swss_docker_name = asichost.get_docker_name("swss")
+
     # Restore the copp template in swss
     if "201811" in dut.os_version or "201911" in dut.os_version:
-        dut.command("docker cp {} {}".format(_BASE_COPP_CONFIG, _APP_DB_COPP_CONFIG))
-        dut.command("docker cp {} {}".format(_TEMP_COPP_TEMPLATE, _SWSS_COPP_TEMPLATE))
+        dut.command("docker cp {} {}".format(_BASE_COPP_CONFIG, swss_docker_name + _APP_DB_COPP_CONFIG))
+        dut.command("docker cp {} {}".format(_TEMP_COPP_TEMPLATE, swss_docker_name + _SWSS_COPP_TEMPLATE))
     else:
         dut.command("cp {} {}".format(_BASE_COPP_CONFIG, _CONFIG_DB_COPP_CONFIG))
 
@@ -114,7 +124,7 @@ def restore_ptf(ptf):
 
     ptf.supervisorctl(name="ptf_nn_agent", state="restarted")
 
-def configure_syncd(dut, nn_target_port, nn_target_interface, creds):
+def configure_syncd(dut, nn_target_port, nn_target_interface, nn_target_namespace, creds):
     """
         Configures syncd to run the NN agent on the specified port.
 
@@ -125,22 +135,38 @@ def configure_syncd(dut, nn_target_port, nn_target_interface, creds):
         Args:
             dut (SonicHost): The target device.
             nn_target_port (int): The port to run NN agent on.
-            nn_target_interface (str): The Interface remote NN agents listens.
+            nn_target_interface (str): The Interface remote NN agents listen to
+            nn_target_namespace (str): The namespace remote NN agents listens
             creds (dict): Credential information according to the dut inventory
     """
 
     facts = {"nn_target_port": nn_target_port, "nn_target_interface": nn_target_interface}
     dut.host.options["variable_manager"].extra_vars.update(facts)
 
-    _install_nano(dut, creds)
+    asichost = dut.asic_instance_from_namespace(nn_target_namespace)
+
+    syncd_docker_name = asichost.get_docker_name("syncd")
+
+    _install_nano(dut, creds, syncd_docker_name)
 
     dut.template(src=_SYNCD_NN_TEMPLATE, dest=_SYNCD_NN_DEST)
-    dut.command("docker cp {} syncd:/etc/supervisor/conf.d/".format(_SYNCD_NN_DEST))
 
-    dut.command("docker exec syncd supervisorctl reread")
-    dut.command("docker exec syncd supervisorctl update")
+    dut.command("docker cp {} {}:/etc/supervisor/conf.d/".format(_SYNCD_NN_DEST, syncd_docker_name))
+
+    dut.command("docker exec {} supervisorctl reread".format(syncd_docker_name))
+    dut.command("docker exec {} supervisorctl update".format(syncd_docker_name))
+
+def restore_syncd(dut, nn_target_namespace):
+    asichost = dut.asic_instance_from_namespace(nn_target_namespace)
 
-def _install_nano(dut, creds):
+    syncd_docker_name = asichost.get_docker_name("syncd")
+
+    dut.command("docker exec {} rm -rf /etc/supervisor/conf.d/{}".format(syncd_docker_name, _SYNCD_NN_FILE))
+    dut.command("docker exec {} supervisorctl reread".format(syncd_docker_name))
+    dut.command("docker exec {} supervisorctl update".format(syncd_docker_name))
+
+
+def _install_nano(dut, creds,  syncd_docker_name):
     """
         Install nanomsg package to syncd container.
 
@@ -149,13 +175,13 @@ def _install_nano(dut, creds):
             creds (dict): Credential information according to the dut inventory
     """
 
-    output = dut.command("docker exec syncd bash -c '[ -d /usr/local/include/nanomsg ] || echo copp'")
+    output = dut.command("docker exec {} bash -c '[ -d /usr/local/include/nanomsg ] || echo copp'".format(syncd_docker_name))
 
     if output["stdout"] == "copp":
         http_proxy = creds.get('proxy_env', {}).get('http_proxy', '')
         https_proxy = creds.get('proxy_env', {}).get('https_proxy', '')
 
-        cmd = '''docker exec -e http_proxy={} -e https_proxy={} syncd bash -c " \
+        cmd = '''docker exec -e http_proxy={} -e https_proxy={} {} bash -c " \
                 rm -rf /var/lib/apt/lists/* \
                 && apt-get update \
                 && apt-get install -y python-pip build-essential libssl-dev python-dev python-setuptools wget cmake \
@@ -165,7 +191,7 @@ def _install_nano(dut, creds):
                 && rm -f 1.0.0.tar.gz && pip2 install cffi==1.7.0 && pip2 install --upgrade cffi==1.7.0 && pip2 install nnpy \
                 && mkdir -p /opt && cd /opt && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py \
                 && mkdir ptf && cd ptf && wget https://raw.githubusercontent.com/p4lang/ptf/master/src/ptf/afpacket.py && touch __init__.py \
-                " '''.format(http_proxy, https_proxy)
+                " '''.format(http_proxy, https_proxy, syncd_docker_name)
         dut.command(cmd)
 
 def _map_port_number_to_interface(dut, nn_target_port):
@@ -175,3 +201,14 @@ def _map_port_number_to_interface(dut, nn_target_port):
 
     interfaces = dut.command("portstat")["stdout_lines"][2:]
     return interfaces[nn_target_port].split()[0]
+
+def _get_http_and_https_proxy_ip(creds):
+    """
+       Get the http and https proxy ip.
+
+       Args:
+           creds (dict): Credential information according to the dut inventory
+    """
+
+    return (re.findall(r'[0-9]+(?:\.[0-9]+){3}', creds.get('proxy_env', {}).get('http_proxy', ''))[0],
+            re.findall(r'[0-9]+(?:\.[0-9]+){3}', creds.get('proxy_env', {}).get('https_proxy', ''))[0])

tests/copp/test_copp.py

@@ -43,7 +43,8 @@
                                   "topo",
                                   "myip",
                                   "peerip",
-                                  "nn_target_interface"])
+                                  "nn_target_interface",
+                                  "nn_target_namespace"])
 _SUPPORTED_PTF_TOPOS = ["ptf32", "ptf64"]
 _SUPPORTED_T1_TOPOS = ["t1", "t1-lag", "t1-64-lag"]
 _TOR_ONLY_PROTOCOL = ["DHCP"]
@@ -112,8 +113,7 @@ def copp_testbed(
     creds,
     ptfhost,
     tbinfo,
-    request,
-    disable_lldp_for_testing  # usefixtures not supported on fixtures
+    request
 ):
     """
         Pytest fixture to handle setup and cleanup for the COPP tests.
@@ -125,10 +125,12 @@ def copp_testbed(
         pytest.skip("Topology not supported by COPP tests")
 
     try:
-        _setup_testbed(duthost, creds, ptfhost, test_params)
+        _setup_multi_asic_proxy(duthost, creds, test_params, tbinfo)
+        _setup_testbed(duthost, creds, ptfhost, test_params, tbinfo)
         yield test_params
     finally:
-        _teardown_testbed(duthost, creds, ptfhost, test_params)
+        _teardown_multi_asic_proxy(duthost, creds, test_params, tbinfo)
+        _teardown_testbed(duthost, creds, ptfhost, test_params, tbinfo)
 
 @pytest.fixture(autouse=True)
 def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
@@ -143,10 +145,6 @@ def ignore_expected_loganalyzer_exceptions(rand_one_dut_hostname, loganalyzer):
             loganalyzer: Loganalyzer utility fixture
     """
     ignoreRegex = [
-        ".*ERR monit.*'lldpd_monitor' process is not running.*",
-        ".*ERR monit.* 'lldp\|lldpd_monitor' status failed.*-- 'lldpd:' is not running.*",
-        ".*ERR monit.*'lldp_syncd' process is not running.*",
-        ".*ERR monit.*'lldp\|lldp_syncd' status failed.*'python2 -m lldp_syncd' is not running.*",
         ".*snmp#snmp-subagent.*",
         ".*kernel reports TIME_ERROR: 0x4041: Clock Unsynchronized.*"
     ]
@@ -207,80 +205,106 @@ def _gather_test_params(tbinfo, duthost, request):
             peerip = bgp_peer["peer_addr"]
             break
 
-    logging.info("nn_target_port {} nn_target_interface {}".format(nn_target_port, nn_target_interface))
+    nn_target_namespace = mg_facts["minigraph_neighbors"][nn_target_interface]['namespace']
+
+    logging.info("nn_target_port {} nn_target_interface {} nn_target_namespace {}".format(nn_target_port, nn_target_interface, nn_target_namespace))
 
     return _COPPTestParameters(nn_target_port=nn_target_port,
                                swap_syncd=swap_syncd,
                                topo=topo,
                                myip=myip,
                                peerip = peerip,
-                               nn_target_interface=nn_target_interface)
+                               nn_target_interface=nn_target_interface,
+                               nn_target_namespace=nn_target_namespace)
 
-def _setup_testbed(dut, creds, ptf, test_params):
+def _setup_testbed(dut, creds, ptf, test_params, tbinfo):
     """
         Sets up the testbed to run the COPP tests.
     """
-
     logging.info("Set up the PTF for COPP tests")
     copp_utils.configure_ptf(ptf, test_params.nn_target_port)
 
     logging.info("Update the rate limit for the COPP policer")
-    copp_utils.limit_policer(dut, _TEST_RATE_LIMIT)
+    copp_utils.limit_policer(dut, _TEST_RATE_LIMIT, test_params.nn_target_namespace)
 
-    if test_params.swap_syncd:
+    # Multi-asic will not support this mode as of now.
+    if test_params.swap_syncd and not dut.is_multi_asic:
         logging.info("Swap out syncd to use RPC image...")
         docker.swap_syncd(dut, creds)
     else:
+        # Set sysctl RCVBUF parameter for tests
+        dut.command("sysctl -w net.core.rmem_max=609430500")
+
+        # Set sysctl SENDBUF parameter for tests
+        dut.command("sysctl -w net.core.wmem_max=609430500")
+
         # NOTE: Even if the rpc syncd image is already installed, we need to restart
         # SWSS for the COPP changes to take effect.
         logging.info("Reloading config and restarting swss...")
         config_reload(dut)
 
     logging.info("Configure syncd RPC for testing")
-    copp_utils.configure_syncd(dut, test_params.nn_target_port, test_params.nn_target_interface, creds)
+    copp_utils.configure_syncd(dut, test_params.nn_target_port, test_params.nn_target_interface, 
+                               test_params.nn_target_namespace, creds)
 
-def _teardown_testbed(dut, creds, ptf, test_params):
+def _teardown_testbed(dut, creds, ptf, test_params, tbinfo):
     """
         Tears down the testbed, returning it to its initial state.
     """
-
     logging.info("Restore PTF post COPP test")
     copp_utils.restore_ptf(ptf)
 
     logging.info("Restore COPP policer to default settings")
-    copp_utils.restore_policer(dut)
+    copp_utils.restore_policer(dut, test_params.nn_target_namespace)
 
-    if test_params.swap_syncd:
+    if test_params.swap_syncd and not dut.is_multi_asic:
         logging.info("Restore default syncd docker...")
         docker.restore_default_syncd(dut, creds)
     else:
+        copp_utils.restore_syncd(dut, test_params.nn_target_namespace)
         logging.info("Reloading config and restarting swss...")
         config_reload(dut)
 
-
-@pytest.fixture(scope="class")
-def disable_lldp_for_testing(
-    duthosts,
-    rand_one_dut_hostname,
-    disable_container_autorestart,
-    enable_container_autorestart
-):
-    """Disables LLDP during testing so that it doesn't interfere with the policer."""
-    duthost = duthosts[rand_one_dut_hostname]
-
-    logging.info("Disabling LLDP for the COPP tests")
-
-    feature_list = ['lldp']
-    disable_container_autorestart(duthost, testcase="test_copp", feature_list=feature_list)
-
-    duthost.command("docker exec lldp supervisorctl stop lldp-syncd")
-    duthost.command("docker exec lldp supervisorctl stop lldpd")
-
-    yield
-
-    logging.info("Restoring LLDP after the COPP tests")
-
-    duthost.command("docker exec lldp supervisorctl start lldpd")
-    duthost.command("docker exec lldp supervisorctl start lldp-syncd")
-
-    enable_container_autorestart(duthost, testcase="test_copp", feature_list=feature_list)
+def _setup_multi_asic_proxy(dut, creds, test_params, tbinfo):
+    """
+        Sets up the testbed to run the COPP tests on multi-asic platfroms via setting proxy.
+    """
+    if not dut.is_multi_asic:
+        return
+
+    logging.info("Adding iptables rules and enabling eth0 port forwarding")
+    http_proxy, https_proxy = copp_utils._get_http_and_https_proxy_ip(creds)
+    # Add IP Table rule for http and ptf nn_agent traffic.
+    dut.command("sudo sysctl net.ipv4.conf.eth0.forwarding=1")
+    mgmt_ip = dut.host.options["inventory_manager"].get_host(dut.hostname).vars["ansible_host"]
+    # Add Rule to communicate to http/s proxy from namespace
+    dut.command("sudo iptables -t nat -A POSTROUTING -p tcp --dport 8080 -j SNAT --to-source {}".format(mgmt_ip))
+    dut.command("sudo ip -n {} rule add from all to {} pref 1 lookup default".format(test_params.nn_target_namespace, http_proxy))
+    if http_proxy != https_proxy:
+        dut.command("sudo ip -n {} rule add from all to {} pref 2 lookup default".format(test_params.nn_target_namespace, https_proxy))
+    # Add Rule to communicate to ptf nn agent client from namespace
+    ns_ip = dut.shell("sudo ip -n {} -4 -o addr show eth0".format(test_params.nn_target_namespace) + " | awk '{print $4}' | cut -d'/' -f1")["stdout"]
+    dut.command("sudo iptables -t nat -A PREROUTING -p tcp --dport 10900 -j DNAT --to-destination {}".format(ns_ip))
+    dut.command("sudo ip -n {} rule add from {} to {} pref 3 lookup default".format(test_params.nn_target_namespace, ns_ip, tbinfo["ptf_ip"]))
+
+def _teardown_multi_asic_proxy(dut, creds, test_params, tbinfo):
+    """
+        Tears down multi asic proxy settings, returning it to its initial state.
+    """
+    if not dut.is_multi_asic:
+        return
+
+    logging.info("Removing iptables rules and disabling eth0 port forwarding")
+    http_proxy, https_proxy = copp_utils._get_http_and_https_proxy_ip(creds)
+    dut.command("sudo sysctl net.ipv4.conf.eth0.forwarding=0")
+    # Delete IP Table rule for http and ptf nn_agent traffic.
+    mgmt_ip = dut.host.options["inventory_manager"].get_host(dut.hostname).vars["ansible_host"]
+    # Delete Rule to communicate to http/s proxy from namespace
+    dut.command("sudo iptables -t nat -D POSTROUTING -p tcp --dport 8080 -j SNAT --to-source {}".format(mgmt_ip))
+    dut.command("sudo ip -n {} rule delete from all to {} pref 1 lookup default".format(test_params.nn_target_namespace, http_proxy))
+    if http_proxy != https_proxy:
+        dut.command("sudo ip -n {} rule delete from all to {} pref 2 lookup default".format(test_params.nn_target_namespace, https_proxy))
+    # Delete Rule to communicate to ptf nn agent client from namespace
+    ns_ip = dut.shell("sudo ip -n {} -4 -o addr show eth0".format(test_params.nn_target_namespace) + " | awk '{print $4}' | cut -d'/' -f1")["stdout"]
+    dut.command("sudo iptables -t nat -D PREROUTING -p tcp --dport 10900 -j DNAT --to-destination {}".format(ns_ip))
+    dut.command("sudo ip -n {} rule delete from {} to {} pref 3 lookup default".format(test_params.nn_target_namespace, ns_ip, tbinfo["ptf_ip"]))