diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 9177338454..db6270a0b7 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -59,9 +59,10 @@ from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS from easybuild.framework.easyconfig.format.one import EB_FORMAT_EXTENSION, retrieve_blocks_in_spec from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT -from easybuild.framework.easyconfig.parser import DEPRECATED_PARAMETERS, REPLACED_PARAMETERS +from easybuild.framework.easyconfig.parser import ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS, REPLACED_PARAMETERS from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig -from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS, TEMPLATE_NAMES_DYNAMIC, template_constant_dict +from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATES, DEPRECATED_TEMPLATES, TEMPLATE_CONSTANTS +from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_DYNAMIC, template_constant_dict from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning, print_msg from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG @@ -118,11 +119,13 @@ def handle_deprecated_or_replaced_easyconfig_parameters(ec_method): def new_ec_method(self, key, *args, **kwargs): """Check whether any replace easyconfig parameters are still used""" # map deprecated parameters to their replacements, issue deprecation warning(/error) - if key in DEPRECATED_PARAMETERS: + if key in ALTERNATE_PARAMETERS: + key = ALTERNATE_PARAMETERS[key] + elif key in DEPRECATED_PARAMETERS: depr_key = key key, ver = DEPRECATED_PARAMETERS[depr_key] _log.deprecated("Easyconfig parameter '%s' is deprecated, use '%s' instead" % (depr_key, key), ver) - if key in REPLACED_PARAMETERS: + elif key in REPLACED_PARAMETERS: _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0') return ec_method(self, key, *args, **kwargs) @@ -179,7 +182,7 @@ def triage_easyconfig_params(variables, ec): for key in variables: # validations are skipped, just set in the config - if key in ec or key in DEPRECATED_PARAMETERS.keys(): + if any(key in d for d in (ec, DEPRECATED_PARAMETERS.keys(), ALTERNATE_PARAMETERS.keys())): ec_params[key] = variables[key] _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key])) elif key in REPLACED_PARAMETERS: @@ -658,7 +661,7 @@ def set_keys(self, params): with self.disable_templating(): for key in sorted(params.keys()): # validations are skipped, just set in the config - if key in self._config.keys() or key in DEPRECATED_PARAMETERS.keys(): + if any(key in x.keys() for x in (self._config, ALTERNATE_PARAMETERS, DEPRECATED_PARAMETERS)): self[key] = params[key] self.log.info("setting easyconfig parameter %s: value %s (type: %s)", key, self[key], type(self[key])) @@ -1994,7 +1997,34 @@ def resolve_template(value, tmpl_dict): try: value = value % tmpl_dict except KeyError: - _log.warning("Unable to resolve template value %s with dict %s", value, tmpl_dict) + # check if any alternate and/or deprecated templates resolve + try: + orig_value = value + # map old templates to new values for alternate and deprecated templates + alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in + ALTERNATE_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} + alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in + ALTERNATE_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()} + depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in + DEPRECATED_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} + + # try templating with alternate and deprecated templates included + value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map} + + for old_tmpl, val in depr_map.items(): + # check which deprecated templates were replaced, and issue deprecation warnings + if old_tmpl in orig_value and val in value: + new_tmpl, ver = DEPRECATED_TEMPLATES[old_tmpl] + _log.deprecated(f"Easyconfig template '{old_tmpl}' is deprecated, use '{new_tmpl}' instead", + ver) + except KeyError: + _log.warning(f"Unable to resolve template value {value} with dict {tmpl_dict}") + + for key in tmpl_dict: + if key in DEPRECATED_TEMPLATES: + new_key, ver = DEPRECATED_TEMPLATES[key] + _log.deprecated(f"Easyconfig template '{key}' is deprecated, use '{new_key}' instead", ver) + else: # this block deals with references to objects and returns other references # for reading this is ok, but for self['x'] = {} diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 8227a40958..9d3189804a 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -39,6 +39,7 @@ from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.framework.easyconfig.format.format import get_format_version, EasyConfigFormat from easybuild.framework.easyconfig.licenses import EASYCONFIG_LICENSES_DICT +from easybuild.framework.easyconfig.templates import ALTERNATE_TEMPLATE_CONSTANTS, DEPRECATED_TEMPLATE_CONSTANTS from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS from easybuild.tools.build_log import EasyBuildError from easybuild.tools.configobj import ConfigObj @@ -86,6 +87,58 @@ def build_easyconfig_variables_dict(): return vars_dict +def handle_deprecated_constants(method): + """Decorator to handle deprecated easyconfig template constants""" + def wrapper(self, key, *args, **kwargs): + """Check whether any deprecated constants are used""" + alternate = ALTERNATE_TEMPLATE_CONSTANTS + deprecated = DEPRECATED_TEMPLATE_CONSTANTS + if key in alternate: + key = alternate[key] + elif key in deprecated: + depr_key = key + key, ver = deprecated[depr_key] + _log.deprecated(f"Easyconfig template constant '{depr_key}' is deprecated, use '{key}' instead", ver) + return method(self, key, *args, **kwargs) + return wrapper + + +class DeprecatedDict(dict): + """Custom dictionary that handles deprecated easyconfig template constants gracefully""" + + def __init__(self, *args, **kwargs): + self.clear() + self.update(*args, **kwargs) + + @handle_deprecated_constants + def __contains__(self, key): + return super().__contains__(key) + + @handle_deprecated_constants + def __delitem__(self, key): + return super().__delitem__(key) + + @handle_deprecated_constants + def __getitem__(self, key): + return super().__getitem__(key) + + @handle_deprecated_constants + def __setitem__(self, key, value): + return super().__setitem__(key, value) + + def update(self, *args, **kwargs): + if args: + if isinstance(args[0], dict): + for key, value in args[0].items(): + self.__setitem__(key, value) + else: + for key, value in args[0]: + self.__setitem__(key, value) + + for key, value in kwargs.items(): + self.__setitem__(key, value) + + class EasyConfigFormatConfigObj(EasyConfigFormat): """ Extended EasyConfig format, with support for a header and sections that are actually parsed (as opposed to exec'ed). @@ -176,7 +229,7 @@ def parse_header(self, header): def parse_pyheader(self, pyheader): """Parse the python header, assign to docstring and cfg""" - global_vars = self.pyheader_env() + global_vars = DeprecatedDict(self.pyheader_env()) self.log.debug("pyheader initial global_vars %s", global_vars) self.log.debug("pyheader text being exec'ed: %s", pyheader) diff --git a/easybuild/framework/easyconfig/parser.py b/easybuild/framework/easyconfig/parser.py index 8f08500815..5cab3b3ddf 100644 --- a/easybuild/framework/easyconfig/parser.py +++ b/easybuild/framework/easyconfig/parser.py @@ -42,6 +42,11 @@ from easybuild.tools.filetools import read_file, write_file +# alternate easyconfig parameters, and their non-deprecated equivalents +ALTERNATE_PARAMETERS = { + # : , +} + # deprecated easyconfig parameters, and their replacements DEPRECATED_PARAMETERS = { # : (, ), diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 388d80f97a..7ee17d2366 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -161,6 +161,26 @@ ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'), ] +# alternate templates, and their equivalents +ALTERNATE_TEMPLATES = { + # : , +} + +# deprecated templates, and their replacements +DEPRECATED_TEMPLATES = { + # : (, ), +} + +# alternate template constants, and their equivalents +ALTERNATE_TEMPLATE_CONSTANTS = { + # : , +} + +# deprecated template constants, and their replacements +DEPRECATED_TEMPLATE_CONSTANTS = { + # : (, ), +} + extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z'] for ext in extensions: suffix = ext.replace('.', '_').upper() diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 274c8deb05..1807ec8d0c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -40,7 +40,6 @@ import textwrap from collections import OrderedDict from easybuild.tools import LooseVersion -from importlib import reload from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -120,6 +119,11 @@ def setUp(self): github_token = gh.fetch_github_token(GITHUB_TEST_ACCOUNT) self.skip_github_tests = github_token is None and os.getenv('FORCE_EB_GITHUB_TESTS') is None + self.orig_easyconfig_DEPRECATED_PARAMETERS = easyconfig.easyconfig.DEPRECATED_PARAMETERS + self.orig_easyconfig_DEPRECATED_TEMPLATES = easyconfig.easyconfig.DEPRECATED_TEMPLATES + self.orig_easyconfig_ALTERNATE_PARAMETERS = easyconfig.easyconfig.ALTERNATE_PARAMETERS + self.orig_easyconfig_ALTERNATE_TEMPLATES = easyconfig.easyconfig.ALTERNATE_TEMPLATES + def prep(self): """Prepare for test.""" # (re)cleanup last test file @@ -133,10 +137,17 @@ def prep(self): def tearDown(self): """ make sure to remove the temporary file """ st.get_cpu_architecture = self.orig_get_cpu_architecture + super(EasyConfigTest, self).tearDown() if os.path.exists(self.eb_file): os.remove(self.eb_file) + # restore orignal values of DEPRECATED_TEMPLATES & co in easyconfig.templates + easyconfig.easyconfig.DEPRECATED_PARAMETERS = self.orig_easyconfig_DEPRECATED_PARAMETERS + easyconfig.easyconfig.DEPRECATED_TEMPLATES = self.orig_easyconfig_DEPRECATED_TEMPLATES + easyconfig.easyconfig.ALTERNATE_PARAMETERS = self.orig_easyconfig_ALTERNATE_PARAMETERS + easyconfig.easyconfig.ALTERNATE_TEMPLATES = self.orig_easyconfig_ALTERNATE_TEMPLATES + def test_empty(self): """ empty files should not parse! """ self.contents = "# empty string" @@ -1451,6 +1462,47 @@ def test_sysroot_template(self): self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) + def test_template_deprecation_and_alternate(self): + """Test deprecation of (and alternate) templates""" + + self.prep() + + template_test_deprecations = { + 'builddir': ('depr_build_dir', '1000000000'), + 'cudaver': ('depr_cuda_ver', '1000000000'), + 'start_dir': ('depr_start_dir', '1000000000'), + } + easyconfig.easyconfig.DEPRECATED_TEMPLATES = template_test_deprecations + + template_test_alternates = { + 'installdir': 'alt_install_dir', + 'version_major_minor': 'alt_ver_maj_min', + } + easyconfig.easyconfig.ALTERNATE_TEMPLATES = template_test_alternates + + tmpl_str = ("cd %(start_dir)s && make %(namelower)s -Dbuild=%(builddir)s --with-cuda='%(cudaver)s'" + " && echo %(alt_install_dir)s %(version_major_minor)s") + tmpl_dict = { + 'depr_build_dir': '/example/build_dir', + 'depr_cuda_ver': '12.1.1', + 'installdir': '/example/installdir', + 'start_dir': '/example/build_dir/start_dir', + 'alt_ver_maj_min': '1.2', + 'namelower': 'foo', + } + + with self.mocked_stdout_stderr() as (_, stderr): + res = resolve_template(tmpl_str, tmpl_dict) + stderr = stderr.getvalue() + + for tmpl in [*template_test_deprecations.keys(), *template_test_alternates.keys()]: + self.assertNotIn("%(" + tmpl + ")s", res) + + for old, (new, ver) in template_test_deprecations.items(): + depr_str = (f"WARNING: Deprecated functionality, will no longer work in EasyBuild v{ver}: " + f"Easyconfig template '{old}' is deprecated, use '{new}' instead") + self.assertIn(depr_str, stderr) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() @@ -1766,6 +1818,52 @@ def foo(key): self.assertErrorRegex(EasyBuildError, error_regex, foo, key) + def test_alternate_easyconfig_parameters(self): + """Test handling of alternate easyconfig parameters.""" + + test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt = test_ec_txt.replace('postinstallcmds', 'post_install_cmds') + test_ec_txt = test_ec_txt.replace('moduleclass', 'env_mod_class') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + # post_install_cmds is not accepted unless it's registered as an alternative easyconfig parameter + easyconfig.easyconfig.ALTERNATE_PARAMETERS = {} + self.assertErrorRegex(EasyBuildError, "post_install_cmds -> postinstallcmds", EasyConfig, test_ec) + + easyconfig.easyconfig.ALTERNATE_PARAMETERS = { + 'env_mod_class': 'moduleclass', + 'post_install_cmds': 'postinstallcmds', + } + ec = EasyConfig(test_ec) + + expected = 'tools' + self.assertEqual(ec['moduleclass'], expected) + self.assertEqual(ec['env_mod_class'], expected) + + expected = ['echo TOY > %(installdir)s/README'] + self.assertEqual(ec['postinstallcmds'], expected) + self.assertEqual(ec['post_install_cmds'], expected) + + # test setting of easyconfig parameter with original & alternate name + ec['moduleclass'] = 'test1' + self.assertEqual(ec['moduleclass'], 'test1') + self.assertEqual(ec['env_mod_class'], 'test1') + ec.update('moduleclass', 'test2') + self.assertEqual(ec['moduleclass'], 'test1 test2 ') + self.assertEqual(ec['env_mod_class'], 'test1 test2 ') + + ec['env_mod_class'] = 'test3' + self.assertEqual(ec['moduleclass'], 'test3') + self.assertEqual(ec['env_mod_class'], 'test3') + ec.update('env_mod_class', 'test4') + self.assertEqual(ec['moduleclass'], 'test3 test4 ') + self.assertEqual(ec['env_mod_class'], 'test3 test4 ') + def test_deprecated_easyconfig_parameters(self): """Test handling of deprecated easyconfig parameters.""" os.environ.pop('EASYBUILD_DEPRECATED') @@ -1775,21 +1873,15 @@ def test_deprecated_easyconfig_parameters(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) - orig_deprecated_parameters = copy.deepcopy(easyconfig.parser.DEPRECATED_PARAMETERS) - easyconfig.parser.DEPRECATED_PARAMETERS.update({ + easyconfig.easyconfig.DEPRECATED_PARAMETERS = { 'foobar': ('barfoo', '0.0'), # deprecated since forever # won't be actually deprecated for a while; # note that we should map foobarbarfoo to a valid easyconfig parameter here, # or we'll hit errors when parsing an easyconfig file that uses it 'foobarbarfoo': ('required_linked_shared_libs', '1000000000'), - }) - - # copy classes before reloading, so we can restore them (otherwise isinstance checks fail) - orig_EasyConfig = copy.deepcopy(easyconfig.easyconfig.EasyConfig) - orig_ActiveMNS = copy.deepcopy(easyconfig.easyconfig.ActiveMNS) - reload(easyconfig.parser) + } - for key, (newkey, depr_ver) in easyconfig.parser.DEPRECATED_PARAMETERS.items(): + for key, (newkey, depr_ver) in easyconfig.easyconfig.DEPRECATED_PARAMETERS.items(): if LooseVersion(depr_ver) <= easybuild.tools.build_log.CURRENT_VERSION: # deprecation error error_regex = "DEPRECATED.*since v%s.*'%s' is deprecated.*use '%s' instead" % (depr_ver, key, newkey) @@ -1802,12 +1894,13 @@ def foo(key): self.assertErrorRegex(EasyBuildError, error_regex, foo, key) else: # only deprecation warning, but key is replaced when getting/setting - ec[key] = 'test123' - self.assertEqual(ec[newkey], 'test123') - self.assertEqual(ec[key], 'test123') - ec[newkey] = '123test' - self.assertEqual(ec[newkey], '123test') - self.assertEqual(ec[key], '123test') + with self.mocked_stdout_stderr(): + ec[key] = 'test123' + self.assertEqual(ec[newkey], 'test123') + self.assertEqual(ec[key], 'test123') + ec[newkey] = '123test' + self.assertEqual(ec[newkey], '123test') + self.assertEqual(ec[key], '123test') variables = { 'name': 'example', @@ -1838,12 +1931,6 @@ def foo(key): ec = EasyConfig(test_ec) self.assertEqual(ec['required_linked_shared_libs'], 'foobarbarfoo') - easyconfig.parser.DEPRECATED_PARAMETERS = orig_deprecated_parameters - reload(easyconfig.parser) - reload(easyconfig.easyconfig) - easyconfig.easyconfig.EasyConfig = orig_EasyConfig - easyconfig.easyconfig.ActiveMNS = orig_ActiveMNS - def test_unknown_easyconfig_parameter(self): """Check behaviour when unknown easyconfig parameters are used.""" self.contents = '\n'.join([