multi-asic support for copp testcase

tests/common/devices/multi_asic.py

@@ -200,6 +200,16 @@ def get_asic(self, asic_id):
             return self.sonichost
         return self.asics[asic_id]
 
+    def get_asic_from_namespace(self, namespace=DEFAULT_NAMESPACE):
+        if not namespace:
+            return self.sonichost
+
+        for asic in self.asics:
+            if asic.namespace == namespace:
+                return asic
+
+        return None
+
     def stop_service(self, service):
         if service in self._DEFAULT_SERVICES:
             return self.sonichost.stop_service(service, service)

tests/copp/copp_utils.py

@@ -12,20 +12,21 @@
 _UPDATE_COPP_SCRIPT = "copp/scripts/update_copp_config.py"
 
 _BASE_COPP_CONFIG = "/tmp/base_copp_config.json"
-_APP_DB_COPP_CONFIG = "swss:/etc/swss/config.d/00-copp.config.json"
+_APP_DB_COPP_CONFIG = ":/etc/swss/config.d/00-copp.config.json"
 _CONFIG_DB_COPP_CONFIG = "/etc/sonic/copp_cfg.json"
 _TEMP_COPP_CONFIG = "/tmp/copp_config.json"
 _TEMP_COPP_TEMPLATE = "/tmp/copp.json.j2"
 _COPP_TEMPLATE_PATH = "/usr/share/sonic/templates/copp.json.j2"
-_SWSS_COPP_TEMPLATE = "swss:" + _COPP_TEMPLATE_PATH
+_SWSS_COPP_TEMPLATE = ":" + _COPP_TEMPLATE_PATH
 
 _PTF_NN_TEMPLATE = "templates/ptf_nn_agent.conf.ptf.j2"
 _PTF_NN_DEST = "/etc/supervisor/conf.d/ptf_nn_agent.conf"
 
 _SYNCD_NN_TEMPLATE = "templates/ptf_nn_agent.conf.dut.j2"
 _SYNCD_NN_DEST = "/tmp/ptf_nn_agent.conf"
+_SYNCD_NN_FILE = "ptf_nn_agent.conf"
 
-def limit_policer(dut, pps_limit):
+def limit_policer(dut, pps_limit, nn_target_namespace):
     """
         Updates the COPP configuration in the SWSS container to respect a given rate limit.
 
@@ -37,8 +38,12 @@ def limit_policer(dut, pps_limit):
             pps_limit (int): The rate limit for COPP to enforce on ALL trap groups.
     """
 
+    asichost = dut.get_asic_from_namespace(nn_target_namespace)
+
+    swss_docker_name = asichost.get_docker_name("swss")
+
     if "201811" in dut.os_version or "201911" in dut.os_version:
-        dut.command("docker cp {} {}".format(_APP_DB_COPP_CONFIG, _BASE_COPP_CONFIG))
+        dut.command("docker cp {} {}".format(swss_docker_name + _APP_DB_COPP_CONFIG, _BASE_COPP_CONFIG))
         config_format = "app_db"
     else:
         dut.command("cp {} {}".format(_CONFIG_DB_COPP_CONFIG, _BASE_COPP_CONFIG))
@@ -53,17 +58,17 @@ def limit_policer(dut, pps_limit):
     )
 
     if config_format == "app_db":
-        dut.command("docker cp {} {}".format(_TEMP_COPP_CONFIG, _APP_DB_COPP_CONFIG))
+        dut.command("docker cp {} {}".format(_TEMP_COPP_CONFIG, swss_docker_name + _APP_DB_COPP_CONFIG))
 
         # As copp config is regenerated each time swss starts need to replace the template with
         # config updated above. But before doing that need store the original template in a
         # temporary file for restore after test.
-        dut.command("docker cp {} {}".format(_SWSS_COPP_TEMPLATE, _TEMP_COPP_TEMPLATE))
-        dut.command("docker cp {} {}".format(_TEMP_COPP_CONFIG, _SWSS_COPP_TEMPLATE))
+        dut.command("docker cp {} {}".format(swss_docker_name + _SWSS_COPP_TEMPLATE, _TEMP_COPP_TEMPLATE))
+        dut.command("docker cp {} {}".format(_TEMP_COPP_CONFIG, swss_docker_name + _SWSS_COPP_TEMPLATE))
     else:
         dut.command("cp {} {}".format(_TEMP_COPP_CONFIG, _CONFIG_DB_COPP_CONFIG))
 
-def restore_policer(dut):
+def restore_policer(dut, nn_target_namespace):
     """
         Reloads the default COPP configuration in the SWSS container.
 
@@ -72,10 +77,14 @@ def restore_policer(dut):
 
             The SWSS container must be restarted for the config change to take effect.
     """
+    asichost = dut.get_asic_from_namespace(nn_target_namespace)
+
+    swss_docker_name = asichost.get_docker_name("swss")
+
     # Restore the copp template in swss
     if "201811" in dut.os_version or "201911" in dut.os_version:
-        dut.command("docker cp {} {}".format(_BASE_COPP_CONFIG, _APP_DB_COPP_CONFIG))
-        dut.command("docker cp {} {}".format(_TEMP_COPP_TEMPLATE, _SWSS_COPP_TEMPLATE))
+        dut.command("docker cp {} {}".format(_BASE_COPP_CONFIG, swss_docker_name + _APP_DB_COPP_CONFIG))
+        dut.command("docker cp {} {}".format(_TEMP_COPP_TEMPLATE, swss_docker_name + _SWSS_COPP_TEMPLATE))
     else:
         dut.command("cp {} {}".format(_BASE_COPP_CONFIG, _CONFIG_DB_COPP_CONFIG))
 
@@ -114,7 +123,7 @@ def restore_ptf(ptf):
 
     ptf.supervisorctl(name="ptf_nn_agent", state="restarted")
 
-def configure_syncd(dut, nn_target_port, nn_target_interface, creds):
+def configure_syncd(dut, nn_target_port, nn_target_interface, nn_target_namespace, creds):
     """
         Configures syncd to run the NN agent on the specified port.
 
@@ -125,22 +134,38 @@ def configure_syncd(dut, nn_target_port, nn_target_interface, creds):
         Args:
             dut (SonicHost): The target device.
             nn_target_port (int): The port to run NN agent on.
-            nn_target_interface (str): The Interface remote NN agents listens.
+            nn_target_interface (str): The Interface remote NN agents listen to
+            nn_target_namespace (str): The namespace remote NN agents listens
             creds (dict): Credential information according to the dut inventory
     """
 
     facts = {"nn_target_port": nn_target_port, "nn_target_interface": nn_target_interface}
     dut.host.options["variable_manager"].extra_vars.update(facts)
 
-    _install_nano(dut, creds)
+    asichost = dut.get_asic_from_namespace(nn_target_namespace)
+
+    syncd_docker_name = asichost.get_docker_name("syncd")
+
+    _install_nano(dut, creds, syncd_docker_name)
 
     dut.template(src=_SYNCD_NN_TEMPLATE, dest=_SYNCD_NN_DEST)
-    dut.command("docker cp {} syncd:/etc/supervisor/conf.d/".format(_SYNCD_NN_DEST))
 
-    dut.command("docker exec syncd supervisorctl reread")
-    dut.command("docker exec syncd supervisorctl update")
+    dut.command("docker cp {} {}:/etc/supervisor/conf.d/".format(_SYNCD_NN_DEST, syncd_docker_name))
+
+    dut.command("docker exec {} supervisorctl reread".format(syncd_docker_name))
+    dut.command("docker exec {} supervisorctl update".format(syncd_docker_name))
+
+def restore_syncd(dut, nn_target_namespace):
+    asichost = dut.get_asic_from_namespace(nn_target_namespace)
+
+    syncd_docker_name = asichost.get_docker_name("syncd")
+
+    dut.command("docker exec {} rm -rf /etc/supervisor/conf.d/{}".format(syncd_docker_name, _SYNCD_NN_FILE))
+    dut.command("docker exec {} supervisorctl reread".format(syncd_docker_name))
+    dut.command("docker exec {} supervisorctl update".format(syncd_docker_name))
+
 
-def _install_nano(dut, creds):
+def _install_nano(dut, creds,  syncd_docker_name):
     """
         Install nanomsg package to syncd container.
 
@@ -149,13 +174,13 @@ def _install_nano(dut, creds):
             creds (dict): Credential information according to the dut inventory
     """
 
-    output = dut.command("docker exec syncd bash -c '[ -d /usr/local/include/nanomsg ] || echo copp'")
+    output = dut.command("docker exec {} bash -c '[ -d /usr/local/include/nanomsg ] || echo copp'".format(syncd_docker_name))
 
     if output["stdout"] == "copp":
         http_proxy = creds.get('proxy_env', {}).get('http_proxy', '')
         https_proxy = creds.get('proxy_env', {}).get('https_proxy', '')
 
-        cmd = '''docker exec -e http_proxy={} -e https_proxy={} syncd bash -c " \
+        cmd = '''docker exec -e http_proxy={} -e https_proxy={} {} bash -c " \
                 rm -rf /var/lib/apt/lists/* \
                 && apt-get update \
                 && apt-get install -y python-pip build-essential libssl-dev python-dev python-setuptools wget cmake \
@@ -165,7 +190,7 @@ def _install_nano(dut, creds):
                 && rm -f 1.0.0.tar.gz && pip2 install cffi==1.7.0 && pip2 install --upgrade cffi==1.7.0 && pip2 install nnpy \
                 && mkdir -p /opt && cd /opt && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py \
                 && mkdir ptf && cd ptf && wget https://raw.githubusercontent.com/p4lang/ptf/master/src/ptf/afpacket.py && touch __init__.py \
-                " '''.format(http_proxy, https_proxy)
+                " '''.format(http_proxy, https_proxy, syncd_docker_name)
         dut.command(cmd)
 
 def _map_port_number_to_interface(dut, nn_target_port):

tests/copp/test_copp.py

@@ -43,7 +43,8 @@
                                   "topo",
                                   "myip",
                                   "peerip",
-                                  "nn_target_interface"])
+                                  "nn_target_interface",
+                                  "nn_target_namespace"])
 _SUPPORTED_PTF_TOPOS = ["ptf32", "ptf64"]
 _SUPPORTED_T1_TOPOS = ["t1", "t1-lag", "t1-64-lag"]
 _TOR_ONLY_PROTOCOL = ["DHCP"]
@@ -214,46 +215,91 @@ def _gather_test_params(tbinfo, duthost, request):
                                topo=topo,
                                myip=myip,
                                peerip = peerip,
-                               nn_target_interface=nn_target_interface)
+                               nn_target_interface=nn_target_interface,
+                               nn_target_namespace=mg_facts["minigraph_neighbors"][nn_target_interface]['namespace'])
 
 def _setup_testbed(dut, creds, ptf, test_params):
     """
         Sets up the testbed to run the COPP tests.
     """
+    if dut.is_multi_asic:
+       logging.info("Adding iptables rules and enabling eth0 port forwarding")
+       # IP Table rule for http and ptf nn_agent traffic.
+       dut.command("sudo sysctl net.ipv4.conf.eth0.forwarding=1")
+       mgmt_ip = dut.host.options["inventory_manager"].get_host(dut.hostname).vars["ansible_host"]
+       dut.command("sudo iptables -t nat -A POSTROUTING -p tcp --dport 8080 -j SNAT --to-source {}".format(mgmt_ip))
+       ip_ifs = dut.show_ip_interface(namespace = test_params.nn_target_namespace)["ansible_facts"]
+       dut.command("sudo iptables -t nat -A PREROUTING -p tcp --dport 10900 -j DNAT --to-destination {}".format(ip_ifs["ip_interfaces"]["eth0"]["ipv4"]))
 
     logging.info("Set up the PTF for COPP tests")
     copp_utils.configure_ptf(ptf, test_params.nn_target_port)
 
     logging.info("Update the rate limit for the COPP policer")
-    copp_utils.limit_policer(dut, _TEST_RATE_LIMIT)
+    copp_utils.limit_policer(dut, _TEST_RATE_LIMIT, test_params.nn_target_namespace)
 
-    if test_params.swap_syncd:
+    # Multi-asic will not support this mode as of now.
+    if test_params.swap_syncd and not dut.is_multi_asic:
         logging.info("Swap out syncd to use RPC image...")
         docker.swap_syncd(dut, creds)
     else:
+        # Set sysctl RCVBUF parameter for tests
+        dut.command("sysctl -w net.core.rmem_max=609430500")
+
+        # Set sysctl SENDBUF parameter for tests
+        dut.command("sysctl -w net.core.wmem_max=609430500")
+
         # NOTE: Even if the rpc syncd image is already installed, we need to restart
         # SWSS for the COPP changes to take effect.
         logging.info("Reloading config and restarting swss...")
-        config_reload(dut)
+        config_reload(dut, wait=180)
+
+    # Shutdown BGP (needed on multi-asic for http connection. keeping generic as of now as it does not imapct testcase functionality) 
+    # and LLDP 
+    logging.info("Disable BGP and LLDP for COPP tests")
+    dut.command("sudo config bgp shutdown all")
+    asichost = dut.get_asic_from_namespace(test_params.nn_target_namespace)
+
+    lldp_docker_name = asichost.get_docker_name("lldp")
+
+    dut.command("docker exec {} supervisorctl stop lldp-syncd".format(lldp_docker_name))
+    dut.command("docker exec {} supervisorctl stop lldpd".format(lldp_docker_name))
 
     logging.info("Configure syncd RPC for testing")
-    copp_utils.configure_syncd(dut, test_params.nn_target_port, test_params.nn_target_interface, creds)
+    copp_utils.configure_syncd(dut, test_params.nn_target_port, test_params.nn_target_interface, 
+                               test_params.nn_target_namespace, creds)
 
 def _teardown_testbed(dut, creds, ptf, test_params):
     """
         Tears down the testbed, returning it to its initial state.
     """
-
+    if dut.is_multi_asic:
+       logging.info("Removing iptables rules and disabling eth0 port forwarding")
+
+       dut.command("sudo sysctl net.ipv4.conf.eth0.forwarding=0")
+
+       mgmt_ip = dut.host.options["inventory_manager"].get_host(dut.hostname).vars["ansible_host"]
+       dut.command("sudo iptables -t nat -D POSTROUTING -p tcp --dport 8080 -j SNAT --to-source {}".format(mgmt_ip))
+
+       ip_ifs = dut.show_ip_interface(namespace = test_params.nn_target_namespace)["ansible_facts"]
+       dut.command("sudo iptables -t nat -D PREROUTING -p tcp --dport 10900 -j DNAT --to-destination {}".format(ip_ifs["ip_interfaces"]["eth0"]["ipv4"]))
+
     logging.info("Restore PTF post COPP test")
     copp_utils.restore_ptf(ptf)
 
     logging.info("Restore COPP policer to default settings")
-    copp_utils.restore_policer(dut)
+    copp_utils.restore_policer(dut, test_params.nn_target_namespace)
 
-    if test_params.swap_syncd:
+    if test_params.swap_syncd and not dut.is_multi_asic:
         logging.info("Restore default syncd docker...")
         docker.restore_default_syncd(dut, creds)
+
+        asichost = dut.get_asic_from_namespace(test_params.nn_target_namespace)
+        lldp_docker_name = asichost.get_docker_name("lldp")
+
+        dut.command("docker exec {} supervisorctl start lldp-syncd".format(lldp_docker_name))
+        dut.command("docker exec {} supervisorctl ssraer lldpd".format(lldp_docker_name))
     else:
+        copp_utils.restore_syncd(dut, test_params.nn_target_namespace)
         logging.info("Reloading config and restarting swss...")
         config_reload(dut)
 
@@ -268,19 +314,13 @@ def disable_lldp_for_testing(
     """Disables LLDP during testing so that it doesn't interfere with the policer."""
     duthost = duthosts[rand_one_dut_hostname]
 
-    logging.info("Disabling LLDP for the COPP tests")
+    logging.info("Disabling LLDP Container auto-restart for the COPP tests")
 
     feature_list = ['lldp']
     disable_container_autorestart(duthost, testcase="test_copp", feature_list=feature_list)
 
-    duthost.command("docker exec lldp supervisorctl stop lldp-syncd")
-    duthost.command("docker exec lldp supervisorctl stop lldpd")
-
     yield
 
     logging.info("Restoring LLDP after the COPP tests")
 
-    duthost.command("docker exec lldp supervisorctl start lldpd")
-    duthost.command("docker exec lldp supervisorctl start lldp-syncd")
-
     enable_container_autorestart(duthost, testcase="test_copp", feature_list=feature_list)