Enable caching of host and group ansible variables

tests/common/cache/facts_cache.py

@@ -44,6 +44,7 @@ class FactsCache(with_metaclass(Singleton, object)):
     def __init__(self, cache_location=CACHE_LOCATION):
         self._cache_location = os.path.abspath(cache_location)
         self._cache = defaultdict(dict)
+        self._write_lock = Lock()
 
     def _check_usage(self):
         """Check cache usage, raise exception if usage exceeds the limitations.
@@ -100,22 +101,23 @@ def write(self, zone, key, value):
         Returns:
             boolean: Caching facts is successful or not.
         """
-        self._check_usage()
-        facts_file = os.path.join(self._cache_location, '{}/{}.pickle'.format(zone, key))
-        try:
-            cache_subfolder = os.path.join(self._cache_location, zone)
-            if not os.path.exists(cache_subfolder):
-                logger.info('Create cache dir {}'.format(cache_subfolder))
-                os.makedirs(cache_subfolder)
-
-            with open(facts_file, 'w') as f:
-                pickle.dump(value, f)
-                self._cache[zone][key] = value
-                logger.info('Cached facts "{}.{}" to {}'.format(zone, key, facts_file))
-                return True
-        except (IOError, ValueError) as e:
-            logger.error('Dump cache file "{}" failed with exception: {}'.format(facts_file, repr(e)))
-            return False
+        with self._write_lock:
+            self._check_usage()
+            facts_file = os.path.join(self._cache_location, '{}/{}.pickle'.format(zone, key))
+            try:
+                cache_subfolder = os.path.join(self._cache_location, zone)
+                if not os.path.exists(cache_subfolder):
+                    logger.info('Create cache dir {}'.format(cache_subfolder))
+                    os.makedirs(cache_subfolder)
+
+                with open(facts_file, 'w') as f:
+                    pickle.dump(value, f)
+                    self._cache[zone][key] = value
+                    logger.info('Cached facts "{}.{}" to {}'.format(zone, key, facts_file))
+                    return True
+            except (IOError, ValueError) as e:
+                logger.error('Dump cache file "{}" failed with exception: {}'.format(facts_file, repr(e)))
+                return False
 
     def cleanup(self, zone=None, key=None):
         """Cleanup cached files.

tests/common/helpers/dut_utils.py

@@ -27,6 +27,3 @@ def is_frontend_node(inv_files, hostname):
           node. If we add more types of nodes, then we need to exclude them from this method as well.
     """
     return not is_supervisor_node(inv_files, hostname)
-
-
-

tests/common/utilities.py

@@ -13,7 +13,10 @@
 from ansible.inventory.manager import InventoryManager
 from ansible.vars.manager import VariableManager
 
+from tests.common.cache import FactsCache
+
 logger = logging.getLogger(__name__)
+cache = FactsCache()
 
 
 def wait(seconds, msg=""):
@@ -137,11 +140,23 @@ def join_all(threads, timeout):
 
 
 def get_inventory_manager(inv_files):
-    return InventoryManager(loader=DataLoader(), sources=inv_files)
+    im_cache = cache.read('common', 'inventory_manager')
+    if im_cache and im_cache['inv_files'] == inv_files:
+        return im_cache['im']
+
+    im = InventoryManager(loader=DataLoader(), sources=inv_files)
+    cache.write('common', 'inventory_manager', {'inv_files': inv_files, 'im': im})
+    return im
 
 
 def get_variable_manager(inv_files):
-    return VariableManager(loader=DataLoader(), inventory=get_inventory_manager(inv_files))
+    vm_cache = cache.read('common', 'variable_manager')
+    if vm_cache and vm_cache['inv_files'] == inv_files:
+        return vm_cache['vm']
+
+    vm = VariableManager(loader=DataLoader(), inventory=get_inventory_manager(inv_files))
+    cache.write('common', 'variable_manager', {'inv_files': inv_files, 'vm': vm})
+    return vm
 
 
 def get_inventory_files(request):
@@ -200,16 +215,25 @@ def get_host_visible_vars(inv_files, hostname, variable=None):
             return None. If variable name is not specified, return all variables in a dictionary. If the host is not
             found, return None.
     """
-    vm = get_variable_manager(inv_files)
-    im = vm._inventory
-    host = im.get_host(hostname)
-    if not host:
-        logger.error("Unable to find host {} in {}".format(hostname, str(inv_files)))
-        return None
+    cached_vars = cache.read(hostname, 'host_visible_vars')
+
+    if cached_vars and cached_vars['inv_files'] == inv_files:
+        host_visible_vars = cached_vars['vars']
+    else:
+        vm = get_variable_manager(inv_files)
+        im = vm._inventory
+        host = im.get_host(hostname)
+        if not host:
+            logger.error("Unable to find host {} in {}".format(hostname, str(inv_files)))
+            return None
+
+        host_visible_vars = vm.get_vars(host=host)
+        cache.write(hostname, 'host_visible_vars', {'inv_files': inv_files, 'vars': host_visible_vars})
+
     if variable:
-        return vm.get_vars(host=host).get(variable, None)
+        return host_visible_vars.get(variable, None)
     else:
-        return vm.get_vars(host=host)
+        return host_visible_vars
 
 
 def get_group_visible_vars(inv_files, group_name, variable=None):
@@ -228,21 +252,29 @@ def get_group_visible_vars(inv_files, group_name, variable=None):
             return None. If variable name is not specified, return all variables in a dictionary. If the group is not
             found or there is no host in the group, return None.
     """
-    vm = get_variable_manager(inv_files)
-    im = vm._inventory
-    group = im.groups.get(group_name, None)
-    if not group:
-        logger.error("Unable to find group {} in {}".format(group_name, str(inv_files)))
-        return None
-    group_hosts = group.get_hosts()
-    if len(group_hosts) == 0:
-        logger.error("No host in group {}".format(group_name))
-        return None
-    first_host = group_hosts[0]
+    cached_vars = cache.read(group_name, 'group_visible_vars')
+    if cached_vars and cached_vars['inv_files'] == inv_files:
+        group_visible_vars = cached_vars['vars']
+    else:
+        vm = get_variable_manager(inv_files)
+        im = vm._inventory
+        group = im.groups.get(group_name, None)
+        if not group:
+            logger.error("Unable to find group {} in {}".format(group_name, str(inv_files)))
+            return None
+        group_hosts = group.get_hosts()
+        if len(group_hosts) == 0:
+            logger.error("No host in group {}".format(group_name))
+            return None
+        first_host = group_hosts[0]
+
+        group_visible_vars = vm.get_vars(host=first_host)
+        cache.write(group_name, 'group_visible_vars', {'inv_files': inv_files, 'vars': group_visible_vars})
+
     if variable:
-        return vm.get_vars(host=first_host).get(variable, None)
+        return group_visible_vars.get(variable, None)
     else:
-        return vm.get_vars(host=first_host)
+        return group_visible_vars
 
 
 def get_test_server_vars(inv_files, server, variable=None):
@@ -265,17 +297,28 @@ def get_test_server_vars(inv_files, server, variable=None):
             return None. If variable name is not specified, return all variables in a dictionary. If the server group
             is not found or there is no test server host in the group, return None.
     """
-    vm = get_variable_manager(inv_files)
-    im = vm._inventory
-    group = im.groups.get(server, None)
-    if not group:
-        logger.error("Unable to find group {} in {}".format(server, str(inv_files)))
+    cached_vars = cache.read(server, 'test_server_vars')
+    if cached_vars and cached_vars['inv_files'] == inv_files:
+        test_server_vars = cached_vars['vars']
+    else:
+        test_server_vars = None
+
+        vm = get_variable_manager(inv_files)
+        im = vm._inventory
+        group = im.groups.get(server, None)
+        if not group:
+            logger.error("Unable to find group {} in {}".format(server, str(inv_files)))
+            return None
+        for host in group.get_hosts():
+            if not re.match(r'VM\d+', host.name):   # This must be the test server host
+                test_server_vars = host.vars
+                cache.write(server, 'test_server_vars', {'inv_files': inv_files, 'vars': test_server_vars})
+
+    if test_server_vars:
+        if variable:
+            return test_server_vars.get(variable, None)
+        else:
+            return test_server_vars
+    else:
+        logger.error("Unable to find test server host under group {}".format(server))
         return None
-    for host in group.get_hosts():
-        if not re.match(r'VM\d+', host.name):   # This must be the test server host
-            if variable:
-                return host.vars.get(variable, None)
-            else:
-                return host.vars
-    logger.error("Unable to find test server host under group {}".format(server))
-    return None

tests/conftest.py

@@ -9,8 +9,6 @@
 import pytest
 import yaml
 import jinja2
-from ansible.parsing.dataloader import DataLoader
-from ansible.inventory.manager import InventoryManager
 
 from datetime import datetime
 from tests.common.fixtures.conn_graph_facts import conn_graph_facts
@@ -20,7 +18,9 @@
 from tests.common.helpers.dut_ports import encode_dut_port_name
 from tests.common.devices import DutHosts
 from tests.common.testbed import TestbedInfo
-from tests.common.utilities import get_inventory_files, get_host_visible_vars
+from tests.common.utilities import get_inventory_files
+from tests.common.utilities import get_host_vars
+from tests.common.utilities import get_host_visible_vars
 from tests.common.helpers.dut_utils import is_supervisor_node, is_frontend_node
 from tests.common.cache import FactsCache
 
@@ -606,14 +606,9 @@ def get_host_data(request, dut):
     '''
     This function parses multple inventory files and returns the dut information present in the inventory
     '''
-    inv_data = None
     inv_files = get_inventory_files(request)
-    for inv_file in inv_files:
-        inv_mgr = InventoryManager(loader=DataLoader(), sources=inv_file)
-        if dut in inv_mgr.hosts:
-            return inv_mgr.get_host(dut).get_vars()
+    return get_host_vars(inv_files, dut)
 
-    return inv_data
 
 def generate_params_frontend_hostname(request, duts_in_testbed, tbname):
     frontend_duts = []
@@ -669,7 +664,7 @@ def generate_params_supervisor_hostname(request, duts_in_testbed, tbname):
 
 
 def generate_param_asic_index(request, duts_in_testbed, dut_indices, param_type):
-    logging.info("generating {} asic indicies for  DUT [{}] in ".format(param_type, dut_indices))
+    logging.info("generating {} asic indices for  DUT [{}] in ".format(param_type, dut_indices))
     #if the params are not present treat the device as a single asic device
     asic_index_params = [DEFAULT_ASIC_ID]
 
@@ -867,4 +862,3 @@ def duthost_console(localhost, creds, request):
                        console_password=creds['console_password'][vars['console_type']])
     yield host
     host.disconnect()
-