diff --git a/tests/common/cache/facts_cache.md b/tests/common/cache/facts_cache.md index a649514128b..cb887b58e05 100644 --- a/tests/common/cache/facts_cache.md +++ b/tests/common/cache/facts_cache.md @@ -35,17 +35,110 @@ When `testbed-cli.sh deploy-mg` is executed for specified testbed, the ansible p There are two ways to use the cache function. ## Use decorator `facts_cache.py::cached` +facts_cache.**cache**(*name, zone_getter=None, after_read=None, before_write=None*) +* This function is a decorator that can be used to cache the result from the decorated function. + * arguments: + * `name`: the key name that result from the decorated function will be stored under. + * `zone_getter`: a function used to find a string that could be used as `zone`, must have three arguments defined: `(function, func_args, func_kargs)`, that `function` is the decorated function, `func_args` and `func_kargs` are those parameters passed the decorated function at runtime. + * `after_read`: a hook function used to process the cached facts after reading from cached file, must have four arguments defined: `(facts, function, func_args, func_kargs)`, `facts` is the just-read cached facts, `function`, `func_args` and `func_kargs` are the same as those in `zone_getter`. + * `before_write`: a hook function used to process the facts returned from decorated function, also must have four arguments defined: `(facts, function, func_args, func_kargs)`. + +### usage +1. default usage to decorate methods in class `AnsibleHostBase` or its derivatives. +```python +from tests.common.cache import cached + +class SonicHost(AnsibleHostBase): + + ... + + @cached(name='basic_facts') + def _gather_facts(self): + ... ``` -from tests.common.cache import cached +2. have custome zone getter function to retrieve zone from the argument `hostname` defined in the decorated function. +```python +import inspect + + +def get_hostname(function, func_args, func_kargs) + args_binding = inspect.getcallargs(function, *func_args, **func_kargs) + return args_binding.get("hostname") or args_binding.get("kargs").get("hostname") + + +@cached(name="host_variable", zone_getter=get_hostname) +def get_host_visible_variable(inv_files, hostname): + pass +``` +3. have custome `after_read` and `before_write` to validate that cached facts are within 24h. +```python +import datetime +import time + + +def validate_datetime_after_read(facts, function, func_args, func_kargs): + if facts is not Facts.NOEXIST: + timestamp = facts.get("cached_timestamp") + if timestamp: + delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(timestamp) + if delta.days == 0: + return facts["cached_facts"] + # if exceeds 24h, force the get the result from calling the decorated function + return FactsCache.NOTEXIST + + +def add_datetime_before_write(facts, function, func_args, func_kargs): + return {"cached_timestamp": time.time(), "cached_facts": facts} + class SonicHost(AnsibleHostBase): -... + ... @cached(name='basic_facts') def _gather_facts(self): -... +``` +2. have custome zone getter function to retrieve zone from the argument `hostname` defined in the decorated function. +```python +import inspect + + +def get_hostname(function, func_args, func_kargs) + args_binding = inspect.getcallargs(function, *func_args, **func_kargs) + return args_binding.get("hostname") or args_binding.get("kargs").get("hostname") + + +@cached(name="host_variable", zone_getter=get_hostname) +def get_host_visible_variable(inv_files, hostname): + pass +``` +3. have custome `after_read` and `before_write` to validate that cached facts are within 24h. +```python +import datetime +import time + + +def validate_datetime_after_read(facts, function, func_args, func_kargs): + timestamp = facts.get("cached_timestamp") + if timestamp: + delta = datetime.datetime.now() - datetime.datetime.fromtimestamp(timestamp) + if delta.days == 0: + return facts["cached_facts"] + # if exceeds 24h, force the get the result from calling the decorated function + return FactsCache.NOTEXIST + + +def add_datetime_before_write(facts, function, func_args, func_kargs): + return {"cached_timestamp": time.time(), "cached_facts": facts} + + +class SonicHost(AnsibleHostBase): + + ... + + @cached(name='basic_facts', after_read=validate_datetime_after_read, before_write=add_datetime_before_write) + def _gather_facts(self): ``` The `cached` decorator supports name argument which correspond to the `key` argument of `read(self, zone, key)` and `write(self, zone, key, value)`. @@ -55,7 +148,7 @@ The `cached` decorator can only be used on an bound method of class which is sub * Import FactsCache and grab the cache instance -``` +```python from tests.common.cache import FactsCache cache = FactsCache() @@ -63,7 +156,7 @@ cache = FactsCache() * Use code like below -``` +```python def get_some_facts(self, *args): cached_facts = cache.read(self.hostname, 'some_facts') @@ -78,7 +171,7 @@ def get_some_facts(self, *args): ``` * Another example -``` +```python def get_something(): info = cache.read('common', 'some_info') if info: diff --git a/tests/common/cache/facts_cache.py b/tests/common/cache/facts_cache.py index f5773a47cb9..01f7d4a743e 100644 --- a/tests/common/cache/facts_cache.py +++ b/tests/common/cache/facts_cache.py @@ -8,9 +8,9 @@ from collections import defaultdict from threading import Lock - from six import with_metaclass + logger = logging.getLogger(__name__) CURRENT_PATH = os.path.realpath(__file__) @@ -41,6 +41,9 @@ class FactsCache(with_metaclass(Singleton, object)): Args: with_metaclass ([function]): Python 2&3 compatible function from the six library for adding metaclass. """ + + NOTEXIST = object() + def __init__(self, cache_location=CACHE_LOCATION): self._cache_location = os.path.abspath(cache_location) self._cache = defaultdict(dict) @@ -87,7 +90,7 @@ def read(self, zone, key): except (IOError, ValueError) as e: logger.info('Load cache file "{}" failed with exception: {}'\ .format(os.path.abspath(facts_file), repr(e))) - return None + return self.NOTEXIST def write(self, zone, key, value): """Store facts to cache. @@ -158,31 +161,57 @@ def cleanup(self, zone=None, key=None): logger.error('Remove cache folder "{}" failed with exception: {}'\ .format(self._cache_location, repr(e))) -def cached(name): + +def _get_hostname_as_zone(function, func_args, func_kargs): + """Default zone getter used for decorator cached.""" + hostname = None + if func_args: + hostname = getattr(func_args[0], "hostname", None) + if not hostname or not isinstance(hostname, str): + raise ValueError("Failed to get attribute 'hostname' of type string from instance of type %s." + % type(func_args[0])) + return hostname + + +def cached(name, zone_getter=None, after_read=None, before_write=None): """Decorator for enabling cache for facts. The cached facts are to be stored by .pickle. Because the cached pickle files must be stored under subfolder - specified by zone, this decorator can only be used for bound method of class which is subclass of AnsibleHostBase. - The classes have attribute 'hostname' that can be used as zone. + specified by zone, the decorate have an option to passed a zone getter function used to get zone. The zone getter + function must have signature of '(function, func_args, func_kargs)' that 'function' is the decorated function, + 'func_args' and 'func_kargs' are the parameters passed to the decorated function at runtime. The zone getter function + should raise an error if it fails to return a string as zone. + With default zone getter function, this decorator can try to find zone: + if the function is a bound method of class AnsibleHostBase and its derivatives, it will try to use its + attribute 'hostname' as zone, or raises an error if 'hostname' doesn't exists or is not a string. Args: name ([str]): Name of the cached facts. - + zone_getter ([function]): Function used to get hostname used as zone. + after_read ([function]): Hook function used to process facts after read from cache. + before_write ([function]): Hook function used to process facts before write into cache. Returns: [function]: Decorator function. """ cache = FactsCache() + def decorator(target): - def wrapper(*args, **kwargs): - hostname = getattr(args[0], 'hostname', None) - if not hostname or not isinstance(hostname, str): - raise Exception('Decorator is only applicable to bound method of class AnsibleHostBase and its sub-classes') - cached_facts = cache.read(hostname, name) - if cached_facts: + def wrapper(*args, **kargs): + _zone_getter = zone_getter or _get_hostname_as_zone + zone = _zone_getter(target, args, kargs) + + cached_facts = cache.read(zone, name) + if after_read: + cached_facts = after_read(cached_facts, target, args, kargs) + if cached_facts is not FactsCache.NOTEXIST: return cached_facts else: - facts = target(*args, **kwargs) - cache.write(hostname, name, facts) + facts = target(*args, **kargs) + if before_write: + _facts = before_write(facts, target, args, kargs) + cache.write(zone, name, _facts) + else: + cache.write(zone, name, facts) return facts return wrapper return decorator diff --git a/tests/common/dualtor/mux_simulator_control.py b/tests/common/dualtor/mux_simulator_control.py index ef9d4fa6058..4d5c3fcf55d 100644 --- a/tests/common/dualtor/mux_simulator_control.py +++ b/tests/common/dualtor/mux_simulator_control.py @@ -29,8 +29,8 @@ def mux_server_url(request, tbinfo): server = tbinfo['server'] vmset_name = tbinfo['group-name'] inv_files = request.config.option.ansible_inventory - ip = utilities.get_test_server_vars(inv_files, server, 'ansible_host') - port = utilities.get_group_visible_vars(inv_files, server, 'mux_simulator_port') + ip = utilities.get_test_server_vars(inv_files, server).get('ansible_host') + port = utilities.get_group_visible_vars(inv_files, server).get('mux_simulator_port') return "http://{}:{}/mux/{}".format(ip, port, vmset_name) @pytest.fixture(scope='module') diff --git a/tests/common/utilities.py b/tests/common/utilities.py index ee5bbe407a1..7cb903aaafb 100644 --- a/tests/common/utilities.py +++ b/tests/common/utilities.py @@ -2,6 +2,7 @@ Utility functions can re-used in testing scripts. """ import collections +import inspect import ipaddress import logging import six @@ -14,6 +15,7 @@ from ansible.inventory.manager import InventoryManager from ansible.vars.manager import VariableManager +from tests.common.cache import cached from tests.common.cache import FactsCache logger = logging.getLogger(__name__) @@ -162,7 +164,48 @@ def get_inventory_files(request): return inv_files -def get_host_vars(inv_files, hostname, variable=None): +def _get_parameter(function, func_args, func_kargs, argname): + """Get the parameter passed as argname to function.""" + args_binding = inspect.getcallargs(function, *func_args, **func_kargs) + return args_binding.get(argname) or args_binding.get("kargs").get(argname) + + +def zone_getter_factory(argname): + """Create zone getter function used to retrieve parameter as zone.""" + + def _zone_getter(function, func_args, func_kargs): + param = _get_parameter(function, func_args, func_kargs, argname) + if param is None: + raise ValueError("Failed to get parameter '%s' from function %s as zone." % (argname, function)) + return param + + return _zone_getter + + +def _check_inv_files_after_read(facts, function, func_args, func_kargs): + """Check if inventory file matches after read host variable from cached files.""" + if facts is not FactsCache.NOTEXIST: + inv_files = _get_parameter(function, func_args, func_kargs, "inv_files") + if inv_files == facts["inv_files"]: + return facts["vars"] + # no facts cached or facts not in the same inventory, return `NOTEXIST` + # to force calling the decorated function to get facts + return FactsCache.NOTEXIST + + +def _mark_inv_files_before_write(facts, function, func_args, func_kargs): + """Add inventory to the facts before write to cached file.""" + inv_files = _get_parameter(function, func_args, func_kargs, "inv_files") + return {"inv_files": inv_files, "vars": facts} + + +@cached( + "host_vars", + zone_getter=zone_getter_factory("hostname"), + after_read=_check_inv_files_after_read, + before_write=_mark_inv_files_before_write +) +def get_host_vars(inv_files, hostname): """Use ansible's InventoryManager to get value of variables defined for the specified host in the specified inventory files. @@ -170,32 +213,25 @@ def get_host_vars(inv_files, hostname, variable=None): inv_files (list or string): List of inventory file pathes, or string of a single inventory file path. In tests, it can be get from request.config.getoption("ansible_inventory"). hostname (string): Hostname - variable (string or None): Variable name. Defaults to None. Returns: - string or dict or None: If variable name is specified, return the variable value. If variable is not found, - return None. If variable name is not specified, return all variables in a dictionary. If the host is not - found, return None. + dict or None: dict if the host is found, None if the host is not found. """ - cached_vars = cache.read(hostname, 'host_vars') - if cached_vars and cached_vars['inv_files'] == inv_files: - host_vars = cached_vars['vars'] - else: - im = get_inventory_manager(inv_files) - host = im.get_host(hostname) - if not host: - logger.error("Unable to find host {} in {}".format(hostname, str(inv_files))) - return None - host_vars = host.vars - cache.write(hostname, 'host_vars', {'inv_files': inv_files, 'vars': host_vars}) - - if variable: - return host_vars.get(variable, None) - else: - return host_vars + im = get_inventory_manager(inv_files) + host = im.get_host(hostname) + if not host: + logger.error("Unable to find host {} in {}".format(hostname, str(inv_files))) + return None + return host.vars.copy() -def get_host_visible_vars(inv_files, hostname, variable=None): +@cached( + "host_visible_vars", + zone_getter=zone_getter_factory("hostname"), + after_read=_check_inv_files_after_read, + before_write=_mark_inv_files_before_write +) +def get_host_visible_vars(inv_files, hostname): """Use ansible's VariableManager and InventoryManager to get value of variables visible to the specified host. The variable could be defined in host_vars or in group_vars that the host belongs to. @@ -203,35 +239,26 @@ def get_host_visible_vars(inv_files, hostname, variable=None): inv_files (list or string): List of inventory file pathes, or string of a single inventory file path. In tests, it can be get from request.config.getoption("ansible_inventory"). hostname (string): Hostname - variable (string or None): Variable name. Defaults to None. Returns: - string or dict or None: If variable name is specified, return the variable value. If variable is not found, - return None. If variable name is not specified, return all variables in a dictionary. If the host is not - found, return None. + dict or None: dict if the host is found, None if the host is not found. """ - cached_vars = cache.read(hostname, 'host_visible_vars') - - if cached_vars and cached_vars['inv_files'] == inv_files: - host_visible_vars = cached_vars['vars'] - else: - vm = get_variable_manager(inv_files) - im = vm._inventory - host = im.get_host(hostname) - if not host: - logger.error("Unable to find host {} in {}".format(hostname, str(inv_files))) - return None - - host_visible_vars = vm.get_vars(host=host) - cache.write(hostname, 'host_visible_vars', {'inv_files': inv_files, 'vars': host_visible_vars}) - - if variable: - return host_visible_vars.get(variable, None) - else: - return host_visible_vars + vm = get_variable_manager(inv_files) + im = vm._inventory + host = im.get_host(hostname) + if not host: + logger.error("Unable to find host {} in {}".format(hostname, str(inv_files))) + return None + return vm.get_vars(host=host) -def get_group_visible_vars(inv_files, group_name, variable=None): +@cached( + "group_visible_vars", + zone_getter=zone_getter_factory("group_name"), + after_read=_check_inv_files_after_read, + before_write=_mark_inv_files_before_write +) +def get_group_visible_vars(inv_files, group_name): """Use ansible's VariableManager and InventoryManager to get value of variables visible to the first host belongs to the specified group. The variable could be defined in host_vars of the first host or in group_vars that the host belongs to. @@ -240,36 +267,22 @@ def get_group_visible_vars(inv_files, group_name, variable=None): inv_files (list or string): List of inventory file pathes, or string of a single inventory file path. In tests, it can be get from request.config.getoption("ansible_inventory"). group_name (string): Name of group in ansible inventory. - variable (string or None): Variable name. Defaults to None. Returns: - string or dict or None: If variable name is specified, return the variable value. If variable is not found, - return None. If variable name is not specified, return all variables in a dictionary. If the group is not - found or there is no host in the group, return None. + dict or None: dict if the host is found, None if the host is not found. """ - cached_vars = cache.read(group_name, 'group_visible_vars') - if cached_vars and cached_vars['inv_files'] == inv_files: - group_visible_vars = cached_vars['vars'] - else: - vm = get_variable_manager(inv_files) - im = vm._inventory - group = im.groups.get(group_name, None) - if not group: - logger.error("Unable to find group {} in {}".format(group_name, str(inv_files))) - return None - group_hosts = group.get_hosts() - if len(group_hosts) == 0: - logger.error("No host in group {}".format(group_name)) - return None - first_host = group_hosts[0] - - group_visible_vars = vm.get_vars(host=first_host) - cache.write(group_name, 'group_visible_vars', {'inv_files': inv_files, 'vars': group_visible_vars}) - - if variable: - return group_visible_vars.get(variable, None) - else: - return group_visible_vars + vm = get_variable_manager(inv_files) + im = vm._inventory + group = im.groups.get(group_name, None) + if not group: + logger.error("Unable to find group {} in {}".format(group_name, str(inv_files))) + return None + group_hosts = group.get_hosts() + if len(group_hosts) == 0: + logger.error("No host in group {}".format(group_name)) + return None + first_host = group_hosts[0] + return vm.get_vars(host=first_host) def get_test_server_host(inv_files, server): @@ -286,7 +299,13 @@ def get_test_server_host(inv_files, server): return None -def get_test_server_vars(inv_files, server, variable=None): +@cached( + "test_server_vars", + zone_getter=zone_getter_factory("server"), + after_read=_check_inv_files_after_read, + before_write=_mark_inv_files_before_write +) +def get_test_server_vars(inv_files, server): """Use ansible's VariableManager and InventoryManager to get value of variables of test server belong to specified server group. @@ -299,34 +318,24 @@ def get_test_server_vars(inv_files, server, variable=None): inv_files (list or string): List of inventory file pathes, or string of a single inventory file path. In tests, it can be get from request.config.getoption("ansible_inventory"). server (string): Server of test setup in testbed.csv file. - variable (string or None): Variable name. Defaults to None. Returns: - string or dict or None: If variable name is specified, return the variable value. If variable is not found, - return None. If variable name is not specified, return all variables in a dictionary. If the server group - is not found or there is no test server host in the group, return None. + dict or None: dict if the host is found, None if the host is not found. """ - cached_vars = cache.read(server, 'test_server_vars') - if cached_vars and cached_vars['inv_files'] == inv_files: - test_server_vars = cached_vars['vars'] - else: - test_server_vars = None - host = get_test_server_host(inv_files, server) - if host: - test_server_vars = host.vars - cache.write(server, 'test_server_vars', {'inv_files': inv_files, 'vars': test_server_vars}) - - if test_server_vars: - if variable: - return test_server_vars.get(variable, None) - else: - return test_server_vars - else: + host = get_test_server_host(inv_files, server) + if not host: logger.error("Unable to find test server host under group {}".format(server)) return None + return host.vars.copy() -def get_test_server_visible_vars(inv_files, server, variable=None): +@cached( + "test_server_visible_vars", + zone_getter=zone_getter_factory("server"), + after_read=_check_inv_files_after_read, + before_write=_mark_inv_files_before_write +) +def get_test_server_visible_vars(inv_files, server): """Use ansible's VariableManager and InventoryManager to get value of variables visible to the specified server group. @@ -339,45 +348,18 @@ def get_test_server_visible_vars(inv_files, server, variable=None): inv_files (list or string): List of inventory file pathes, or string of a single inventory file path. In tests, it can be get from request.config.getoption("ansible_inventory"). server (string): Server of test setup in testbed.csv file. - variable (string or None): Variable name. Defaults to None. Returns: - string or dict or None: If variable name is specified, return the variable value. If variable is not found, - return None. If variable name is not specified, return all variables in a dictionary. If the group is not - found or there is no host in the group, return None. + dict or None: dict if the host is found, None if the host is not found. """ - cached_vars = cache.read(server, 'test_server_visible_vars') - if cached_vars and cached_vars['inv_files'] == inv_files: - test_server_visible_vars = cached_vars['vars'] - else: - test_server_visible_vars = None - - vm = get_variable_manager(inv_files) - im = vm._inventory - group = im.groups.get(server, None) - if not group: - logger.error("Unable to find group {} in {}".format(server, str(inv_files))) - return None - for host in group.get_hosts(): - if not re.match(r'VM\d+', host.name): # This must be the test server host - test_server = host.name - test_server_host = im.get_host(test_server) - if not test_server_host: - logger.error("Unable to find host %s in %s", test_server_host, inv_files) - return None - - test_server_visible_vars = vm.get_vars(host=test_server_host) - cache.write(server, 'test_server_visible_vars', {'inv_files': inv_files, 'vars': test_server_visible_vars}) - - if test_server_visible_vars: - if variable: - return test_server_visible_vars.get(variable, None) - else: - return test_server_visible_vars - else: - logger.error("Unable to find test server host under group {}".format(server)) + test_server_host = get_test_server_host(inv_files, server) + vm = get_variable_manager(inv_files) + if not test_server_host: + logger.error("Unable to find host %s in %s", test_server_host, inv_files) return None + return vm.get_vars(host=test_server_host) + def is_ipv4_address(ip_address): """Check if ip address is ipv4.""" diff --git a/tests/conftest.py b/tests/conftest.py index 02edf64a3e6..d4039371be4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -162,7 +162,7 @@ def get_tbinfo(request): raise ValueError("testbed and testbed_file are required!") testbedinfo = cache.read(tbname, 'tbinfo') - if not testbedinfo: + if testbedinfo is cache.NOTEXIST: testbedinfo = TestbedInfo(tbfile) cache.write(tbname, 'tbinfo', testbedinfo)