Parallel manager2

ansible/ansible.cfg

@@ -124,7 +124,8 @@ connection_plugins = plugins/connection
 lookup_plugins = plugins/lookup
 # vars_plugins       = /usr/share/ansible_plugins/vars_plugins
 filter_plugins     = plugins/filter
-callback_whitelist = profile_tasks
+# Disable profile tasks callback to avoid possible deadlock
+# callback_whitelist = profile_tasks
 
 # by default callbacks are not loaded for /bin/ansible, enable this if you
 # want, for example, a notification or logging callback to also apply to
@@ -190,7 +191,7 @@ become_ask_pass=False
 # ssh arguments to use
 # Leaving off ControlPersist will result in poor performance, so use
 # paramiko on older platforms rather than removing it
-ssh_args = -o ControlMaster=auto -o ControlPersist=7200s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=70
+ssh_args = -o ControlMaster=auto -o ControlPersist=7200s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ServerAliveInterval=30 -o ServerAliveCountMax=70 -o TCPKeepAlive=yes
 
 
 # The path to use for the ControlPath sockets. This defaults to

tests/common/devices/base.py

@@ -55,29 +55,28 @@ def __init__(self, ansible_adhoc, hostname, *args, **kwargs):
 
     def __getattr__(self, module_name):
         if self.host.has_module(module_name):
-            self.module_name = module_name
-            self.module = getattr(self.host, module_name)
-
-            return self._run
+            def _run_wrapper(*module_args, **kwargs):
+                return self._run(module_name, *module_args, **kwargs)
+            return _run_wrapper
         raise AttributeError(
             "'%s' object has no attribute '%s'" % (self.__class__, module_name)
             )
 
-    def _run(self, *module_args, **complex_args):
+    def _run(self, module_name, *module_args, **complex_args):
 
         previous_frame = inspect.currentframe().f_back
         filename, line_number, function_name, lines, index = inspect.getframeinfo(previous_frame)
 
         verbose = complex_args.pop('verbose', True)
-
+        module = getattr(self.host, module_name)
         if verbose:
             logger.debug(
                 "{}::{}#{}: [{}] AnsibleModule::{}, args={}, kwargs={}".format(
                     filename,
                     function_name,
                     line_number,
                     self.hostname,
-                    self.module_name,
+                    module_name,
                     json.dumps(module_args, cls=AnsibleHostBase.CustomEncoder),
                     json.dumps(complex_args, cls=AnsibleHostBase.CustomEncoder)
                 )
@@ -89,7 +88,7 @@ def _run(self, *module_args, **complex_args):
                     function_name,
                     line_number,
                     self.hostname,
-                    self.module_name
+                    module_name
                 )
             )
 
@@ -98,17 +97,17 @@ def _run(self, *module_args, **complex_args):
 
         if module_async:
             def run_module(module_args, complex_args):
-                return self.module(*module_args, **complex_args)[self.hostname]
+                return module(*module_args, **complex_args)[self.hostname]
             pool = ThreadPool()
             result = pool.apply_async(run_module, (module_args, complex_args))
             return pool, result
 
         module_args = json.loads(json.dumps(module_args, cls=AnsibleHostBase.CustomEncoder))
         complex_args = json.loads(json.dumps(complex_args, cls=AnsibleHostBase.CustomEncoder))
 
-        adhoc_res: AdHocResult = self.module(*module_args, **complex_args)
+        adhoc_res: AdHocResult = module(*module_args, **complex_args)
 
-        if self.module_name == "meta":
+        if module_name == "meta":
             # The meta module is special in Ansible - it doesn't execute on remote hosts, it controls Ansible's behavior
             # There are no per-host ModuleResults contained within it
             return
@@ -123,7 +122,7 @@ def run_module(module_args, complex_args):
                     function_name,
                     line_number,
                     self.hostname,
-                    self.module_name, json.dumps(hostname_res, cls=AnsibleHostBase.CustomEncoder)
+                    module_name, json.dumps(hostname_res, cls=AnsibleHostBase.CustomEncoder)
                 )
             )
         else:
@@ -133,14 +132,14 @@ def run_module(module_args, complex_args):
                     function_name,
                     line_number,
                     self.hostname,
-                    self.module_name,
+                    module_name,
                     hostname_res.is_failed,
                     hostname_res.get('rc', None)
                 )
             )
 
         if (hostname_res.is_failed or 'exception' in hostname_res) and not module_ignore_errors:
-            raise RunAnsibleModuleFail("run module {} failed".format(self.module_name), hostname_res)
+            raise RunAnsibleModuleFail("run module {} failed".format(module_name), hostname_res)
 
         return hostname_res
 

tests/common/devices/duthosts.py

@@ -19,9 +19,9 @@ class DutHosts(object):
     """
     class _Nodes(list):
         """ Internal class representing a list of MultiAsicSonicHosts """
-        def _run_on_nodes(self, *module_args, **complex_args):
+        def _run_on_nodes(self, module, *module_args, **complex_args):
             """ Delegate the call to each of the nodes, return the results in a dict."""
-            return {node.hostname: getattr(node, self.attr)(*module_args, **complex_args) for node in self}
+            return {node.hostname: getattr(node, module)(*module_args, **complex_args) for node in self}
 
         def __getattr__(self, attr):
             """ To support calling ansible modules on a list of MultiAsicSonicHost
@@ -32,8 +32,9 @@ def __getattr__(self, attr):
                a dictionary with key being the MultiAsicSonicHost's hostname,
                and value being the output of ansible module on that MultiAsicSonicHost
             """
-            self.attr = attr
-            return self._run_on_nodes
+            def _run_on_nodes_wrapper(*module_args, **complex_args):
+                return self._run_on_nodes(attr, *module_args, **complex_args)
+            return _run_on_nodes_wrapper
 
         def __eq__(self, o):
             """ To support eq operator on the DUTs (nodes) in the testbed """
@@ -62,6 +63,12 @@ def __init__(self, ansible_adhoc, tbinfo, request, duts, target_hostname=None, i
         self.request = request
         self.duts = duts
         self.is_parallel_run = target_hostname is not None
+        # Initialize _nodes to None to avoid recursion in __getattr__
+        self._nodes = None
+        self._nodes_for_parallel = None
+        self._supervisor_nodes = None
+        self._frontend_nodes = None
+
         # TODO: Initialize the nodes in parallel using multi-threads?
         if self.is_parallel_run:
             self.parallel_run_stage = NON_INITIAL_CHECKS_STAGE

tests/common/devices/multi_asic.py

@@ -123,10 +123,11 @@ def critical_services_tracking_list(self):
     def get_default_critical_services_list(self):
         return self._DEFAULT_SERVICES
 
-    def _run_on_asics(self, *module_args, **complex_args):
+    def _run_on_asics(self, multi_asic_attr, *module_args, **complex_args):
         """ Run an asible module on asics based on 'asic_index' keyword in complex_args
 
         Args:
+            multi_asic_attr: name of the ansible module to run
             module_args: other ansible module args passed from the caller
             complex_args: other ansible keyword args
 
@@ -147,7 +148,7 @@ def _run_on_asics(self, *module_args, **complex_args):
         """
         if "asic_index" not in complex_args:
             # Default ASIC/namespace
-            return getattr(self.sonichost, self.multi_asic_attr)(*module_args, **complex_args)
+            return getattr(self.sonichost, multi_asic_attr)(*module_args, **complex_args)
         else:
             asic_complex_args = copy.deepcopy(complex_args)
             asic_index = asic_complex_args.pop("asic_index")
@@ -156,11 +157,11 @@ def _run_on_asics(self, *module_args, **complex_args):
                 if self.sonichost.facts['num_asic'] == 1:
                     if asic_index != 0:
                         raise ValueError("Trying to run module '{}' against asic_index '{}' on a single asic dut '{}'"
-                                         .format(self.multi_asic_attr, asic_index, self.sonichost.hostname))
-                return getattr(self.asic_instance(asic_index), self.multi_asic_attr)(*module_args, **asic_complex_args)
+                                         .format(multi_asic_attr, asic_index, self.sonichost.hostname))
+                return getattr(self.asic_instance(asic_index), multi_asic_attr)(*module_args, **asic_complex_args)
             elif type(asic_index) == str and asic_index.lower() == "all":
                 # All ASICs/namespace
-                return [getattr(asic, self.multi_asic_attr)(*module_args, **asic_complex_args) for asic in self.asics]
+                return [getattr(asic, multi_asic_attr)(*module_args, **asic_complex_args) for asic in self.asics]
             else:
                 raise ValueError("Argument 'asic_index' must be an int or string 'all'.")
 
@@ -357,8 +358,9 @@ def __getattr__(self, attr):
         """
         sonic_asic_attr = getattr(SonicAsic, attr, None)
         if not attr.startswith("_") and sonic_asic_attr and callable(sonic_asic_attr):
-            self.multi_asic_attr = attr
-            return self._run_on_asics
+            def _run_on_asics_wrapper(*module_args, **complex_args):
+                return self._run_on_asics(attr, *module_args, **complex_args)
+            return _run_on_asics_wrapper
         else:
             return getattr(self.sonichost, attr)  # For backward compatibility