diff --git a/tests/common/cache/facts_cache.py b/tests/common/cache/facts_cache.py index 5525741e89c..8b61e8dbf49 100644 --- a/tests/common/cache/facts_cache.py +++ b/tests/common/cache/facts_cache.py @@ -44,6 +44,7 @@ class FactsCache(with_metaclass(Singleton, object)): def __init__(self, cache_location=CACHE_LOCATION): self._cache_location = os.path.abspath(cache_location) self._cache = defaultdict(dict) + self._write_lock = Lock() def _check_usage(self): """Check cache usage, raise exception if usage exceeds the limitations. @@ -100,22 +101,23 @@ def write(self, zone, key, value): Returns: boolean: Caching facts is successful or not. """ - self._check_usage() - facts_file = os.path.join(self._cache_location, '{}/{}.pickle'.format(zone, key)) - try: - cache_subfolder = os.path.join(self._cache_location, zone) - if not os.path.exists(cache_subfolder): - logger.info('Create cache dir {}'.format(cache_subfolder)) - os.makedirs(cache_subfolder) - - with open(facts_file, 'w') as f: - pickle.dump(value, f) - self._cache[zone][key] = value - logger.info('Cached facts "{}.{}" to {}'.format(zone, key, facts_file)) - return True - except (IOError, ValueError) as e: - logger.error('Dump cache file "{}" failed with exception: {}'.format(facts_file, repr(e))) - return False + with self._write_lock: + self._check_usage() + facts_file = os.path.join(self._cache_location, '{}/{}.pickle'.format(zone, key)) + try: + cache_subfolder = os.path.join(self._cache_location, zone) + if not os.path.exists(cache_subfolder): + logger.info('Create cache dir {}'.format(cache_subfolder)) + os.makedirs(cache_subfolder) + + with open(facts_file, 'w') as f: + pickle.dump(value, f) + self._cache[zone][key] = value + logger.info('Cached facts "{}.{}" to {}'.format(zone, key, facts_file)) + return True + except (IOError, ValueError) as e: + logger.error('Dump cache file "{}" failed with exception: {}'.format(facts_file, repr(e))) + return False def cleanup(self, zone=None, key=None): """Cleanup cached files. diff --git a/tests/common/helpers/dut_utils.py b/tests/common/helpers/dut_utils.py index c54e7aede34..00304a94cb9 100644 --- a/tests/common/helpers/dut_utils.py +++ b/tests/common/helpers/dut_utils.py @@ -27,6 +27,3 @@ def is_frontend_node(inv_files, hostname): node. If we add more types of nodes, then we need to exclude them from this method as well. """ return not is_supervisor_node(inv_files, hostname) - - - diff --git a/tests/common/utilities.py b/tests/common/utilities.py index 0679a3ef5b8..cb357d1b442 100644 --- a/tests/common/utilities.py +++ b/tests/common/utilities.py @@ -13,7 +13,10 @@ from ansible.inventory.manager import InventoryManager from ansible.vars.manager import VariableManager +from tests.common.cache import FactsCache + logger = logging.getLogger(__name__) +cache = FactsCache() def wait(seconds, msg=""): @@ -137,11 +140,23 @@ def join_all(threads, timeout): def get_inventory_manager(inv_files): - return InventoryManager(loader=DataLoader(), sources=inv_files) + im_cache = cache.read('common', 'inventory_manager') + if im_cache and im_cache['inv_files'] == inv_files: + return im_cache['im'] + + im = InventoryManager(loader=DataLoader(), sources=inv_files) + cache.write('common', 'inventory_manager', {'inv_files': inv_files, 'im': im}) + return im def get_variable_manager(inv_files): - return VariableManager(loader=DataLoader(), inventory=get_inventory_manager(inv_files)) + vm_cache = cache.read('common', 'variable_manager') + if vm_cache and vm_cache['inv_files'] == inv_files: + return vm_cache['vm'] + + vm = VariableManager(loader=DataLoader(), inventory=get_inventory_manager(inv_files)) + cache.write('common', 'variable_manager', {'inv_files': inv_files, 'vm': vm}) + return vm def get_inventory_files(request): @@ -200,16 +215,25 @@ def get_host_visible_vars(inv_files, hostname, variable=None): return None. If variable name is not specified, return all variables in a dictionary. If the host is not found, return None. """ - vm = get_variable_manager(inv_files) - im = vm._inventory - host = im.get_host(hostname) - if not host: - logger.error("Unable to find host {} in {}".format(hostname, str(inv_files))) - return None + cached_vars = cache.read(hostname, 'host_visible_vars') + + if cached_vars and cached_vars['inv_files'] == inv_files: + host_visible_vars = cached_vars['vars'] + else: + vm = get_variable_manager(inv_files) + im = vm._inventory + host = im.get_host(hostname) + if not host: + logger.error("Unable to find host {} in {}".format(hostname, str(inv_files))) + return None + + host_visible_vars = vm.get_vars(host=host) + cache.write(hostname, 'host_visible_vars', {'inv_files': inv_files, 'vars': host_visible_vars}) + if variable: - return vm.get_vars(host=host).get(variable, None) + return host_visible_vars.get(variable, None) else: - return vm.get_vars(host=host) + return host_visible_vars def get_group_visible_vars(inv_files, group_name, variable=None): @@ -228,21 +252,29 @@ def get_group_visible_vars(inv_files, group_name, variable=None): return None. If variable name is not specified, return all variables in a dictionary. If the group is not found or there is no host in the group, return None. """ - vm = get_variable_manager(inv_files) - im = vm._inventory - group = im.groups.get(group_name, None) - if not group: - logger.error("Unable to find group {} in {}".format(group_name, str(inv_files))) - return None - group_hosts = group.get_hosts() - if len(group_hosts) == 0: - logger.error("No host in group {}".format(group_name)) - return None - first_host = group_hosts[0] + cached_vars = cache.read(group_name, 'group_visible_vars') + if cached_vars and cached_vars['inv_files'] == inv_files: + group_visible_vars = cached_vars['vars'] + else: + vm = get_variable_manager(inv_files) + im = vm._inventory + group = im.groups.get(group_name, None) + if not group: + logger.error("Unable to find group {} in {}".format(group_name, str(inv_files))) + return None + group_hosts = group.get_hosts() + if len(group_hosts) == 0: + logger.error("No host in group {}".format(group_name)) + return None + first_host = group_hosts[0] + + group_visible_vars = vm.get_vars(host=first_host) + cache.write(group_name, 'group_visible_vars', {'inv_files': inv_files, 'vars': group_visible_vars}) + if variable: - return vm.get_vars(host=first_host).get(variable, None) + return group_visible_vars.get(variable, None) else: - return vm.get_vars(host=first_host) + return group_visible_vars def get_test_server_vars(inv_files, server, variable=None): @@ -265,17 +297,28 @@ def get_test_server_vars(inv_files, server, variable=None): return None. If variable name is not specified, return all variables in a dictionary. If the server group is not found or there is no test server host in the group, return None. """ - vm = get_variable_manager(inv_files) - im = vm._inventory - group = im.groups.get(server, None) - if not group: - logger.error("Unable to find group {} in {}".format(server, str(inv_files))) + cached_vars = cache.read(server, 'test_server_vars') + if cached_vars and cached_vars['inv_files'] == inv_files: + test_server_vars = cached_vars['vars'] + else: + test_server_vars = None + + vm = get_variable_manager(inv_files) + im = vm._inventory + group = im.groups.get(server, None) + if not group: + logger.error("Unable to find group {} in {}".format(server, str(inv_files))) + return None + for host in group.get_hosts(): + if not re.match(r'VM\d+', host.name): # This must be the test server host + test_server_vars = host.vars + cache.write(server, 'test_server_vars', {'inv_files': inv_files, 'vars': test_server_vars}) + + if test_server_vars: + if variable: + return test_server_vars.get(variable, None) + else: + return test_server_vars + else: + logger.error("Unable to find test server host under group {}".format(server)) return None - for host in group.get_hosts(): - if not re.match(r'VM\d+', host.name): # This must be the test server host - if variable: - return host.vars.get(variable, None) - else: - return host.vars - logger.error("Unable to find test server host under group {}".format(server)) - return None diff --git a/tests/conftest.py b/tests/conftest.py index 9a340a1ba46..1398367590e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,8 +9,6 @@ import pytest import yaml import jinja2 -from ansible.parsing.dataloader import DataLoader -from ansible.inventory.manager import InventoryManager from datetime import datetime from tests.common.fixtures.conn_graph_facts import conn_graph_facts @@ -20,7 +18,9 @@ from tests.common.helpers.dut_ports import encode_dut_port_name from tests.common.devices import DutHosts from tests.common.testbed import TestbedInfo -from tests.common.utilities import get_inventory_files, get_host_visible_vars +from tests.common.utilities import get_inventory_files +from tests.common.utilities import get_host_vars +from tests.common.utilities import get_host_visible_vars from tests.common.helpers.dut_utils import is_supervisor_node, is_frontend_node from tests.common.cache import FactsCache @@ -606,14 +606,9 @@ def get_host_data(request, dut): ''' This function parses multple inventory files and returns the dut information present in the inventory ''' - inv_data = None inv_files = get_inventory_files(request) - for inv_file in inv_files: - inv_mgr = InventoryManager(loader=DataLoader(), sources=inv_file) - if dut in inv_mgr.hosts: - return inv_mgr.get_host(dut).get_vars() + return get_host_vars(inv_files, dut) - return inv_data def generate_params_frontend_hostname(request, duts_in_testbed, tbname): frontend_duts = [] @@ -669,7 +664,7 @@ def generate_params_supervisor_hostname(request, duts_in_testbed, tbname): def generate_param_asic_index(request, duts_in_testbed, dut_indices, param_type): - logging.info("generating {} asic indicies for DUT [{}] in ".format(param_type, dut_indices)) + logging.info("generating {} asic indices for DUT [{}] in ".format(param_type, dut_indices)) #if the params are not present treat the device as a single asic device asic_index_params = [DEFAULT_ASIC_ID] @@ -867,4 +862,3 @@ def duthost_console(localhost, creds, request): console_password=creds['console_password'][vars['console_type']]) yield host host.disconnect() -