diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 123e4f7695..b781bf2218 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -3921,7 +3921,7 @@ def update_config_template_run_step(self): """Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names""" for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - self.cfg.template_values[name[0]] = str(getattr(self, name[0], None)) + self.cfg.template_values[name] = str(getattr(self, name, None)) self.cfg.generate_template_values() def skip_step(self, step, skippable): diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index ae678f0269..baaaa16af0 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1217,7 +1217,7 @@ def dump(self, fp, always_overwrite=True, backup=False, explicit_toolchains=Fals default_values.update({key: value[0] for key, value in self.extra_options.items()}) self.generate_template_values() - templ_const = {quote_py_str(const[1]): const[0] for const in TEMPLATE_CONSTANTS} + templ_const = {quote_py_str(value): name for name, (value, _) in TEMPLATE_CONSTANTS.items()} # create reverse map of templates, to inject template values where possible # longer template values are considered first, shorter template keys get preference over longer ones @@ -1842,7 +1842,7 @@ def get_cuda_cc_template_value(self, key): Returns user-friendly error message in case neither are defined, or if an unknown key is used. """ - if key.startswith('cuda_') and any(x[0] == key for x in TEMPLATE_NAMES_DYNAMIC): + if key.startswith('cuda_') and any(x == key for x in TEMPLATE_NAMES_DYNAMIC): try: return self.template_values[key] except KeyError: diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 48e6516e8c..c52a459322 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -52,8 +52,8 @@ def build_easyconfig_constants_dict(): """Make a dictionary with all constants that can be used""" all_consts = [ - ('TEMPLATE_CONSTANTS', {x[0]: x[1] for x in TEMPLATE_CONSTANTS}), - ('EASYCONFIG_CONSTANTS', {key: val[0] for key, val in EASYCONFIG_CONSTANTS.items()}), + ('TEMPLATE_CONSTANTS', {name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()}), + ('EASYCONFIG_CONSTANTS', {name: value for name, (value, _) in EASYCONFIG_CONSTANTS.items()}), ('EASYCONFIG_LICENSES', {klass().name: name for name, klass in EASYCONFIG_LICENSES_DICT.items()}), ] err = [] diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index a5dc888369..57ce5d061a 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -45,15 +45,15 @@ _log = fancylogger.getLogger('easyconfig.templates', fname=False) # derived from easyconfig, but not from ._config directly -TEMPLATE_NAMES_EASYCONFIG = [ - ('module_name', "Module name"), - ('nameletter', "First letter of software name"), - ('toolchain_name', "Toolchain name"), - ('toolchain_version', "Toolchain version"), - ('version_major_minor', "Major.Minor version"), - ('version_major', "Major version"), - ('version_minor', "Minor version"), -] +TEMPLATE_NAMES_EASYCONFIG = { + 'module_name': 'Module name', + 'nameletter': 'First letter of software name', + 'toolchain_name': 'Toolchain name', + 'toolchain_version': 'Toolchain version', + 'version_major_minor': "Major.Minor version", + 'version_major': 'Major version', + 'version_minor': 'Minor version', +} # derived from EasyConfig._config TEMPLATE_NAMES_CONFIG = [ 'bitbucket_account', @@ -71,100 +71,105 @@ 'nameletter', ] # values taken from the EasyBlock before each step -TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = [ - ('builddir', "Build directory"), - ('installdir', "Installation directory"), - ('start_dir', "Directory in which the build process begins"), -] +TEMPLATE_NAMES_EASYBLOCK_RUN_STEP = { + 'builddir': 'Build directory', + 'installdir': 'Installation directory', + 'start_dir': 'Directory in which the build process begins', +} # software names for which to define ver, majver and shortver templates -TEMPLATE_SOFTWARE_VERSIONS = [ - # software name, prefix for *ver, *majver and *shortver - ('CUDA', 'cuda'), - ('CUDAcore', 'cuda'), - ('Java', 'java'), - ('Perl', 'perl'), - ('Python', 'py'), - ('R', 'r'), -] +TEMPLATE_SOFTWARE_VERSIONS = { + # software name -> prefix for *ver, *majver and *shortver + 'CUDA': 'cuda', + 'CUDAcore': 'cuda', + 'Java': 'java', + 'Perl': 'perl', + 'Python': 'py', + 'R': 'r', +} # template values which are only generated dynamically -TEMPLATE_NAMES_DYNAMIC = [ - ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), - ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" - "as specified by the --sysroot configuration option"), - ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), - ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " - "--cuda-compute-capabilities configuration option or via cuda_cc easyconfig parameter"), - ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), - ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), - ('cuda_cc_space_sep_no_period', - "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90')."), - ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), - ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"), - ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"), - ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"), - ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), - ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), - ('software_commit', "Git commit id to use for the software as specified by --software-commit command line option"), -] +TEMPLATE_NAMES_DYNAMIC = { + 'arch': 'System architecture (e.g. x86_64, aarch64, ppc64le, ...)', + 'sysroot': "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" + "as specify by the --sysroot configuration option", + 'mpi_cmd_prefix': 'Prefix command for running MPI programs (with default number of ranks)', + 'cuda_compute_capabilities': "Comma-separated list of CUDA compute capabilities, as specified via " + "--cuda-compute-capabilities configuration option or " + "via cuda_compute_capabilities easyconfig parameter", + 'cuda_cc_cmake': 'List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+', + 'cuda_cc_space_sep': 'Space-separated list of CUDA compute capabilities', + 'cuda_cc_space_sep_no_period': + "Space-separated list of CUDA compute capabilities, without periods (e.g. '80 90').", + 'cuda_cc_semicolon_sep': 'Semicolon-separated list of CUDA compute capabilities', + 'cuda_int_comma_sep': 'Comma-separated list of integer CUDA compute capabilities', + 'cuda_int_space_sep': 'Space-separated list of integer CUDA compute capabilities', + 'cuda_int_semicolon_sep': 'Semicolon-separated list of integer CUDA compute capabilities', + 'cuda_sm_comma_sep': 'Comma-separated list of sm_* values that correspond with CUDA compute capabilities', + 'cuda_sm_space_sep': 'Space-separated list of sm_* values that correspond with CUDA compute capabilities', +} # constant templates that can be used in easyconfigs -TEMPLATE_CONSTANTS = [ +# Entry: constant -> (value, doc) +TEMPLATE_CONSTANTS = { # source url constants - ('APACHE_SOURCE', 'https://archive.apache.org/dist/%(namelower)s', - 'apache.org source url'), - ('BITBUCKET_SOURCE', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get', - 'bitbucket.org source url (namelower is used if bitbucket_account easyconfig parameter is not specified)'), - ('BITBUCKET_DOWNLOADS', 'https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads', - 'bitbucket.org downloads url (namelower is used if bitbucket_account easyconfig parameter is not specified)'), - ('CRAN_SOURCE', 'https://cran.r-project.org/src/contrib', - 'CRAN (contrib) source url'), - ('FTPGNOME_SOURCE', 'https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s', - 'http download for gnome ftp server'), - ('GITHUB_SOURCE', 'https://github.com/%(github_account)s/%(name)s/archive', - 'GitHub source URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'), - ('GITHUB_LOWER_SOURCE', 'https://github.com/%(github_account)s/%(namelower)s/archive', - 'GitHub source URL with lowercase name (if github_account easyconfig ' - 'parameter is not specified, namelower is used in its place)'), - ('GITHUB_RELEASE', 'https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s', - 'GitHub release URL (if github_account easyconfig parameter is not specified, namelower is used in its place)'), - ('GITHUB_LOWER_RELEASE', 'https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s', - 'GitHub release URL with lowercase name (if github_account easyconfig ' - 'parameter is not specified, namelower is used in its place)'), - ('GNU_SAVANNAH_SOURCE', 'https://download-mirror.savannah.gnu.org/releases/%(namelower)s', - 'download.savannah.gnu.org source url'), - ('GNU_SOURCE', 'https://ftpmirror.gnu.org/gnu/%(namelower)s', - 'gnu.org source url (ftp mirror)'), - ('GNU_FTP_SOURCE', 'https://ftp.gnu.org/gnu/%(namelower)s', - 'gnu.org source url (main ftp)'), - ('GOOGLECODE_SOURCE', 'http://%(namelower)s.googlecode.com/files', - 'googlecode.com source url'), - ('LAUNCHPAD_SOURCE', 'https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', - 'launchpad.net source url'), - ('PYPI_SOURCE', 'https://pypi.python.org/packages/source/%(nameletter)s/%(name)s', - 'pypi source url'), # e.g., Cython, Sphinx - ('PYPI_LOWER_SOURCE', 'https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', - 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ - ('R_SOURCE', 'https://cran.r-project.org/src/base/R-%(version_major)s', - 'cran.r-project.org (base) source url'), - ('SOURCEFORGE_SOURCE', 'https://download.sourceforge.net/%(namelower)s', - 'sourceforge.net source url'), - ('XORG_DATA_SOURCE', 'https://xorg.freedesktop.org/archive/individual/data/', - 'xorg data source url'), - ('XORG_LIB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/lib/', - 'xorg lib source url'), - ('XORG_PROTO_SOURCE', 'https://xorg.freedesktop.org/archive/individual/proto/', - 'xorg proto source url'), - ('XORG_UTIL_SOURCE', 'https://xorg.freedesktop.org/archive/individual/util/', - 'xorg util source url'), - ('XORG_XCB_SOURCE', 'https://xorg.freedesktop.org/archive/individual/xcb/', - 'xorg xcb source url'), + 'APACHE_SOURCE': ('https://archive.apache.org/dist/%(namelower)s', + 'apache.org source url'), + 'BITBUCKET_SOURCE': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/get', + 'bitbucket.org source url ' + '(namelower is used if bitbucket_account easyconfig parameter is not specified)'), + 'BITBUCKET_DOWNLOADS': ('https://bitbucket.org/%(bitbucket_account)s/%(namelower)s/downloads', + 'bitbucket.org downloads url ' + '(namelower is used if bitbucket_account easyconfig parameter is not specified)'), + 'CRAN_SOURCE': ('https://cran.r-project.org/src/contrib', + 'CRAN (contrib) source url'), + 'FTPGNOME_SOURCE': ('https://ftp.gnome.org/pub/GNOME/sources/%(namelower)s/%(version_major_minor)s', + 'http download for gnome ftp server'), + 'GITHUB_SOURCE': ('https://github.com/%(github_account)s/%(name)s/archive', + 'GitHub source URL ' + '(namelower is used if github_account easyconfig parameter is not specified)'), + 'GITHUB_LOWER_SOURCE': ('https://github.com/%(github_account)s/%(namelower)s/archive', + 'GitHub source URL with lowercase name ' + '(namelower is used if github_account easyconfig parameter is not specified)'), + 'GITHUB_RELEASE': ('https://github.com/%(github_account)s/%(name)s/releases/download/v%(version)s', + 'GitHub release URL ' + '(namelower is use if github_account easyconfig parameter is not specified)'), + 'GITHUB_LOWER_RELEASE': ('https://github.com/%(github_account)s/%(namelower)s/releases/download/v%(version)s', + 'GitHub release URL with lowercase name (if github_account easyconfig ' + 'parameter is not specified, namelower is used in its place)'), + 'GNU_SAVANNAH_SOURCE': ('https://download-mirror.savannah.gnu.org/releases/%(namelower)s', + 'download.savannah.gnu.org source url'), + 'GNU_SOURCE': ('https://ftpmirror.gnu.org/gnu/%(namelower)s', + 'gnu.org source url (ftp mirror)'), + 'GNU_FTP_SOURCE': ('https://ftp.gnu.org/gnu/%(namelower)s', + 'gnu.org source url (main ftp)'), + 'GOOGLECODE_SOURCE': ('http://%(namelower)s.googlecode.com/files', + 'googlecode.com source url'), + 'LAUNCHPAD_SOURCE': ('https://launchpad.net/%(namelower)s/%(version_major_minor)s.x/%(version)s/+download/', + 'launchpad.net source url'), + 'PYPI_SOURCE': ('https://pypi.python.org/packages/source/%(nameletter)s/%(name)s', + 'pypi source url'), # e.g., Cython, Sphinx + 'PYPI_LOWER_SOURCE': ('https://pypi.python.org/packages/source/%(nameletterlower)s/%(namelower)s', + 'pypi source url (lowercase name)'), # e.g., Greenlet, PyZMQ + 'R_SOURCE': ('https://cran.r-project.org/src/base/R-%(version_major)s', + 'cran.r-project.org (base) source url'), + 'SOURCEFORGE_SOURCE': ('https://download.sourceforge.net/%(namelower)s', + 'sourceforge.net source url'), + 'XORG_DATA_SOURCE': ('https://xorg.freedesktop.org/archive/individual/data/', + 'xorg data source url'), + 'XORG_LIB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/lib/', + 'xorg lib source url'), + 'XORG_PROTO_SOURCE': ('https://xorg.freedesktop.org/archive/individual/proto/', + 'xorg proto source url'), + 'XORG_UTIL_SOURCE': ('https://xorg.freedesktop.org/archive/individual/util/', + 'xorg util source url'), + 'XORG_XCB_SOURCE': ('https://xorg.freedesktop.org/archive/individual/xcb/', + 'xorg xcb source url'), # TODO, not urgent, yet nice to have: # CPAN_SOURCE GNOME KDE_I18N XCONTRIB DEBIAN KDE GENTOO TEX_CTAN MOZILLA_ALL # other constants - ('SHLIB_EXT', get_shared_lib_ext(), 'extension for shared libraries'), -] + 'SHLIB_EXT': (get_shared_lib_ext(), 'extension for shared libraries'), +} # alternative templates, and their equivalents ALTERNATIVE_EASYCONFIG_TEMPLATES = { @@ -172,11 +177,8 @@ 'build_dir': 'builddir', 'cuda_cc_comma_sep': 'cuda_compute_capabilities', 'cuda_maj_ver': 'cudamajver', - 'cuda_maj_ver': 'cudamajver', - 'cuda_short_ver': 'cudashortver', 'cuda_short_ver': 'cudashortver', 'cuda_ver': 'cudaver', - 'cuda_ver': 'cudaver', 'install_dir': 'installdir', 'java_maj_ver': 'javamajver', 'java_short_ver': 'javashortver', @@ -254,13 +256,13 @@ # : (, ), } -extensions = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z'] -for ext in extensions: +EXTENSIONS = ['tar.gz', 'tar.xz', 'tar.bz2', 'tgz', 'txz', 'tbz2', 'tb2', 'gtgz', 'zip', 'tar', 'xz', 'tar.Z'] +for ext in EXTENSIONS: suffix = ext.replace('.', '_').upper() - TEMPLATE_CONSTANTS += [ - ('SOURCE_%s' % suffix, '%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext), - ('SOURCELOWER_%s' % suffix, '%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext), - ] + TEMPLATE_CONSTANTS.update({ + 'SOURCE_%s' % suffix: ('%(name)s-%(version)s.' + ext, "Source .%s bundle" % ext), + 'SOURCELOWER_%s' % suffix: ('%(namelower)s-%(version)s.' + ext, "Source .%s bundle with lowercase name" % ext), + }) for pyver in ('py2.py3', 'py2', 'py3'): if pyver == 'py2.py3': desc = 'Python 2 & Python 3' @@ -268,12 +270,12 @@ else: desc = 'Python ' + pyver[-1] name_infix = pyver.upper() + '_' - TEMPLATE_CONSTANTS += [ - ('SOURCE_%sWHL' % name_infix, '%%(name)s-%%(version)s-%s-none-any.whl' % pyver, - 'Generic (non-compiled) %s wheel package' % desc), - ('SOURCELOWER_%sWHL' % name_infix, '%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver, - 'Generic (non-compiled) %s wheel package with lowercase name' % desc), - ] + TEMPLATE_CONSTANTS.update({ + 'SOURCE_%sWHL' % name_infix: ('%%(name)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package' % desc), + 'SOURCELOWER_%sWHL' % name_infix: ('%%(namelower)s-%%(version)s-%s-none-any.whl' % pyver, + 'Generic (non-compiled) %s wheel package with lowercase name' % desc), + }) # TODO derived config templates # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) @@ -281,10 +283,9 @@ def template_constant_dict(config, ignore=None, toolchain=None): """Create a dict for templating the values in the easyconfigs. - - config is a dict with the structure of EasyConfig._config + - config -- Dict with the structure of EasyConfig._config + - ignore -- List of template names to ignore """ - # TODO find better name - # ignore if ignore is None: ignore = [] # make dict @@ -307,10 +308,10 @@ def template_constant_dict(config, ignore=None, toolchain=None): continue # check if this template name is already handled - if template_values.get(name[0]) is not None: + if template_values.get(name) is not None: continue - if name[0].startswith('toolchain_'): + if name.startswith('toolchain_'): tc = config.get('toolchain') if tc is not None: template_values['toolchain_name'] = tc.get('name', None) @@ -318,7 +319,7 @@ def template_constant_dict(config, ignore=None, toolchain=None): # only go through this once ignore.extend(['toolchain_name', 'toolchain_version']) - elif name[0].startswith('version_'): + elif name.startswith('version_'): # parse major and minor version numbers version = config['version'] if version is not None: @@ -337,87 +338,85 @@ def template_constant_dict(config, ignore=None, toolchain=None): # only go through this once ignore.extend(['version_major', 'version_minor', 'version_major_minor']) - elif name[0].endswith('letter'): + elif name.endswith('letter'): # parse first letters - if name[0].startswith('name'): + if name.startswith('name'): softname = config['name'] if softname is not None: template_values['nameletter'] = softname[0] - elif name[0] == 'module_name': + elif name == 'module_name': template_values['module_name'] = getattr(config, 'short_mod_name', None) else: raise EasyBuildError("Undefined name %s from TEMPLATE_NAMES_EASYCONFIG", name) # step 2: define *ver and *shortver templates - if TEMPLATE_SOFTWARE_VERSIONS: - - name_to_prefix = {name.lower(): pref for name, pref in TEMPLATE_SOFTWARE_VERSIONS} - deps = config.get('dependencies', []) - - # also consider build dependencies for *ver and *shortver templates; - # we need to be a bit careful here, because for iterative installations - # (when multi_deps is used for example) the builddependencies value may be a list of lists - - # first, determine if we have an EasyConfig instance - # (indirectly by checking for 'iterating' and 'iterate_options' attributes, - # because we can't import the EasyConfig class here without introducing - # a cyclic import...); - # we need to know to determine whether we're iterating over a list of build dependencies - is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options') - if is_easyconfig: - # if we're iterating over different lists of build dependencies, - # only consider build dependencies when we're actually in iterative mode! - if 'builddependencies' in config.iterate_options: - if config.iterating: - build_deps = config.get('builddependencies') - else: - build_deps = None - else: + name_to_prefix = {name.lower(): prefix for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items()} + deps = config.get('dependencies', []) + + # also consider build dependencies for *ver and *shortver templates; + # we need to be a bit careful here, because for iterative installations + # (when multi_deps is used for example) the builddependencies value may be a list of lists + + # first, determine if we have an EasyConfig instance + # (indirectly by checking for 'iterating' and 'iterate_options' attributes, + # because we can't import the EasyConfig class here without introducing + # a cyclic import...); + # we need to know to determine whether we're iterating over a list of build dependencies + is_easyconfig = hasattr(config, 'iterating') and hasattr(config, 'iterate_options') + if is_easyconfig: + # if we're iterating over different lists of build dependencies, + # only consider build dependencies when we're actually in iterative mode! + if 'builddependencies' in config.iterate_options: + if config.iterating: build_deps = config.get('builddependencies') + else: + build_deps = None + else: + build_deps = config.get('builddependencies') + if build_deps: + # Don't use += to avoid changing original list + deps = deps + build_deps + # include all toolchain deps (e.g. CUDAcore component in fosscuda); + # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain! + if config._toolchain is not None and config._toolchain.tcdeps: + # If we didn't create a new list above do it here if build_deps: - # Don't use += to avoid changing original list - deps = deps + build_deps - # include all toolchain deps (e.g. CUDAcore component in fosscuda); - # access Toolchain instance via _toolchain to avoid triggering initialization of the toolchain! - if config._toolchain is not None and config._toolchain.tcdeps: - # If we didn't create a new list above do it here - if build_deps: - deps.extend(config._toolchain.tcdeps) - else: - deps = deps + config._toolchain.tcdeps - - for dep in deps: - if isinstance(dep, dict): - dep_name, dep_version = dep['name'], dep['version'] - - # take into account dependencies marked as external modules, - # where name/version may have to be harvested from metadata available for that external module - if dep.get('external_module', False): - metadata = dep.get('external_module_metadata', {}) - if dep_name is None: - # name is a list in metadata, just take first value (if any) - dep_name = metadata.get('name', [None])[0] - if dep_version is None: - # version is a list in metadata, just take first value (if any) - dep_version = metadata.get('version', [None])[0] - - elif isinstance(dep, (list, tuple)): - dep_name, dep_version = dep[0], dep[1] + deps.extend(config._toolchain.tcdeps) else: - raise EasyBuildError("Unexpected type for dependency: %s", dep) - - if isinstance(dep_name, str) and dep_version: - pref = name_to_prefix.get(dep_name.lower()) - if pref: - dep_version = pick_dep_version(dep_version) - template_values['%sver' % pref] = dep_version - dep_version_parts = dep_version.split('.') - template_values['%smajver' % pref] = dep_version_parts[0] - if len(dep_version_parts) > 1: - template_values['%sminver' % pref] = dep_version_parts[1] - template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) + deps = deps + config._toolchain.tcdeps + + for dep in deps: + if isinstance(dep, dict): + dep_name, dep_version = dep['name'], dep['version'] + + # take into account dependencies marked as external modules, + # where name/version may have to be harvested from metadata available for that external module + if dep.get('external_module', False): + metadata = dep.get('external_module_metadata', {}) + if dep_name is None: + # name is a list in metadata, just take first value (if any) + dep_name = metadata.get('name', [None])[0] + if dep_version is None: + # version is a list in metadata, just take first value (if any) + dep_version = metadata.get('version', [None])[0] + + elif isinstance(dep, (list, tuple)): + dep_name, dep_version = dep[0], dep[1] + else: + raise EasyBuildError("Unexpected type for dependency: %s", dep) + + if isinstance(dep_name, str) and dep_version: + pref = name_to_prefix.get(dep_name.lower()) + if pref: + dep_version = pick_dep_version(dep_version) + template_values['%sver' % pref] = dep_version + dep_version_parts = dep_version.split('.') + template_values['%smajver' % pref] = dep_version_parts[0] + if len(dep_version_parts) > 1: + template_values['%sminver' % pref] = dep_version_parts[1] + template_values['%sshortver' % pref] = '.'.join(dep_version_parts[:2]) # step 3: add remaining from config for name in TEMPLATE_NAMES_CONFIG: @@ -475,8 +474,7 @@ def template_constant_dict(config, ignore=None, toolchain=None): unknown_names = [] for key in template_values: - dynamic_template_names = set(x for (x, _) in TEMPLATE_NAMES_DYNAMIC) - if not (key in common_template_names or key in dynamic_template_names): + if not (key in common_template_names or key in TEMPLATE_NAMES_DYNAMIC): unknown_names.append(key) if unknown_names: raise EasyBuildError("One or more template values found with unknown name: %s", ','.join(unknown_names)) @@ -526,15 +524,15 @@ def template_documentation(): # step 1: add TEMPLATE_NAMES_EASYCONFIG doc.append('Template names/values derived from easyconfig instance') - for name in TEMPLATE_NAMES_EASYCONFIG: - doc.append("%s%%(%s)s: %s" % (indent_l1, name[0], name[1])) + for name, cur_doc in TEMPLATE_NAMES_EASYCONFIG.items(): + doc.append("%s%%(%s)s: %s" % (indent_l1, name, cur_doc)) # step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS doc.append("Template names/values for (short) software versions") - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: - doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name)) - doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, pref, name)) - doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name)) + for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items(): + doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, prefix, name)) + doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, prefix, name)) + doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, prefix, name)) # step 3: add remaining self._config doc.append('Template names/values as set in easyconfig') @@ -550,11 +548,16 @@ def template_documentation(): # step 5. self.template_values can/should be updated from outside easyconfig # (eg the run_setp code in EasyBlock) doc.append('Template values set outside EasyBlock runstep') - for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - doc.append("%s%%(%s)s: %s" % (indent_l1, name[0], name[1])) + for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items(): + doc.append("%s%%(%s)s: %s" % (indent_l1, name, cur_doc)) doc.append('Template constants that can be used in easyconfigs') - for cst in TEMPLATE_CONSTANTS: - doc.append('%s%s: %s (%s)' % (indent_l1, cst[0], cst[2], cst[1])) + for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items(): + doc.append('%s%s: %s (%s)' % (indent_l1, name, cur_doc, value)) return "\n".join(doc) + + +# Add template constants to export list +globals().update({name: value for name, (value, _) in TEMPLATE_CONSTANTS.items()}) +__all__ = list(TEMPLATE_CONSTANTS.keys()) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index c6020706d5..90b9abaac2 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -116,7 +116,7 @@ def __init__(self, mself, ext, extra_params=None): # Add install/builddir templates with values from master. for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None)) + self.cfg.template_values[key] = str(getattr(self.master, key, None)) # We can't inherit the 'start_dir' value from the parent (which will be set, and will most likely be wrong). # It should be specified for the extension specifically, or be empty (so it is auto-derived). diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 4bbdefa66b..69e8d17337 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -478,16 +478,16 @@ def avail_easyconfig_templates_txt(): # step 1: add TEMPLATE_NAMES_EASYCONFIG doc.append('Template names/values derived from easyconfig instance') - for name in TEMPLATE_NAMES_EASYCONFIG: - doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + for name, curDoc in TEMPLATE_NAMES_EASYCONFIG.items(): + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, curDoc)) doc.append('') # step 2: add SOFTWARE_VERSIONS doc.append('Template names/values for (short) software versions') - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: - doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name)) - doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, pref, name)) - doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name)) + for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items(): + doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, prefix, name)) + doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, prefix, name)) + doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, prefix, name)) doc.append('') # step 3: add remaining config @@ -506,20 +506,20 @@ def avail_easyconfig_templates_txt(): # step 5: template_values can/should be updated from outside easyconfig # (eg the run_step code in EasyBlock) doc.append('Template values set outside EasyBlock runstep') - for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: - doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + for name, cur_doc in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.items(): + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc)) doc.append('') # some template values are only defined dynamically, # see template_constant_dict function in easybuild.framework.easyconfigs.templates doc.append('Template values which are defined dynamically') - for name in TEMPLATE_NAMES_DYNAMIC: - doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name[0], name[1])) + for name, cur_doc in TEMPLATE_NAMES_DYNAMIC.items(): + doc.append("%s%%(%s)s: %s" % (INDENT_4SPACES, name, cur_doc)) doc.append('') doc.append('Template constants that can be used in easyconfigs') - for cst in TEMPLATE_CONSTANTS: - doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, cst[0], cst[2], cst[1])) + for name, (value, cur_doc) in TEMPLATE_CONSTANTS.items(): + doc.append('%s%s: %s (%s)' % (INDENT_4SPACES, name, cur_doc, value)) return '\n'.join(doc) @@ -530,8 +530,8 @@ def avail_easyconfig_templates_rst(): title = 'Template names/values derived from easyconfig instance' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG], - [name[1] for name in TEMPLATE_NAMES_EASYCONFIG], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG], + list(TEMPLATE_NAMES_EASYCONFIG.values()), ] doc = rst_title_and_table(title, table_titles, table_values) doc.append('') @@ -539,10 +539,10 @@ def avail_easyconfig_templates_rst(): title = 'Template names/values for (short) software versions' ver = [] ver_desc = [] - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: - ver.append('``%%(%smajver)s``' % pref) - ver.append('``%%(%sshortver)s``' % pref) - ver.append('``%%(%sver)s``' % pref) + for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items(): + ver.append('``%%(%smajver)s``' % prefix) + ver.append('``%%(%sshortver)s``' % prefix) + ver.append('``%%(%sver)s``' % prefix) ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (.)' % name) ver_desc.append('full version for %s' % name) @@ -565,24 +565,24 @@ def avail_easyconfig_templates_rst(): title = 'Template values set outside EasyBlock runstep' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], - [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()), ] doc.extend(rst_title_and_table(title, table_titles, table_values)) title = 'Template values which are defined dynamically' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], - [name[1] for name in TEMPLATE_NAMES_DYNAMIC], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC], + list(TEMPLATE_NAMES_DYNAMIC.values()), ] doc.extend(rst_title_and_table(title, table_titles, table_values)) title = 'Template constants that can be used in easyconfigs' - titles = ['Constant', 'Template value', 'Template name'] + titles = ['Constant', 'Template description', 'Template value'] table_values = [ - ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS], - [cst[2] for cst in TEMPLATE_CONSTANTS], - ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS], + ['``%s``' % name for name in TEMPLATE_CONSTANTS], + [doc for _, doc in TEMPLATE_CONSTANTS.values()], + ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()], ] doc.extend(rst_title_and_table(title, titles, table_values)) @@ -595,8 +595,8 @@ def avail_easyconfig_templates_md(): title = 'Template names/values derived from easyconfig instance' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYCONFIG], - [name[1] for name in TEMPLATE_NAMES_EASYCONFIG], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYCONFIG], + list(TEMPLATE_NAMES_EASYCONFIG.values()), ] doc = md_title_and_table(title, table_titles, table_values, title_level=2) doc.append('') @@ -604,10 +604,10 @@ def avail_easyconfig_templates_md(): title = 'Template names/values for (short) software versions' ver = [] ver_desc = [] - for name, pref in TEMPLATE_SOFTWARE_VERSIONS: - ver.append('``%%(%smajver)s``' % pref) - ver.append('``%%(%sshortver)s``' % pref) - ver.append('``%%(%sver)s``' % pref) + for name, prefix in TEMPLATE_SOFTWARE_VERSIONS.items(): + ver.append('``%%(%smajver)s``' % prefix) + ver.append('``%%(%sshortver)s``' % prefix) + ver.append('``%%(%sver)s``' % prefix) ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (``.``)' % name) ver_desc.append('full version for %s' % name) @@ -631,27 +631,28 @@ def avail_easyconfig_templates_md(): title = 'Template values set outside EasyBlock runstep' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], - [name[1] for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP], + list(TEMPLATE_NAMES_EASYBLOCK_RUN_STEP.values()), ] doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') title = 'Template values which are defined dynamically' table_values = [ - ['``%%(%s)s``' % name[0] for name in TEMPLATE_NAMES_DYNAMIC], - [name[1] for name in TEMPLATE_NAMES_DYNAMIC], + ['``%%(%s)s``' % name for name in TEMPLATE_NAMES_DYNAMIC], + list(TEMPLATE_NAMES_DYNAMIC.values()), ] doc.extend(md_title_and_table(title, table_titles, table_values, title_level=2)) doc.append('') title = 'Template constants that can be used in easyconfigs' - titles = ['Constant', 'Template value', 'Template name'] + titles = ['Constant', 'Template description', 'Template value'] table_values = [ - ['``%s``' % cst[0] for cst in TEMPLATE_CONSTANTS], - [cst[2] for cst in TEMPLATE_CONSTANTS], - ['``%s``' % cst[1] for cst in TEMPLATE_CONSTANTS], + ['``%s``' % name for name in TEMPLATE_CONSTANTS], + [doc for _, doc in TEMPLATE_CONSTANTS.values()], + ['``%s``' % value for value, _ in TEMPLATE_CONSTANTS.values()], ] + doc.extend(md_title_and_table(title, titles, table_values, title_level=2)) return '\n'.join(doc) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index 87ee6381a2..2875d2a42c 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -1271,6 +1271,14 @@ def test_templating_constants(self): ec = EasyConfig(test_ec) self.assertEqual(ec['sanity_check_commands'], ['mpiexec -np 1 -- toy']) + def test_template_constant_import(self): + """Test importing template constants works""" + from easybuild.framework.easyconfig.templates import GITHUB_SOURCE, GNU_SOURCE, SHLIB_EXT + from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS + self.assertEqual(GITHUB_SOURCE, TEMPLATE_CONSTANTS['GITHUB_SOURCE'][0]) + self.assertEqual(GNU_SOURCE, TEMPLATE_CONSTANTS['GNU_SOURCE'][0]) + self.assertEqual(SHLIB_EXT, get_shared_lib_ext()) + def test_templating_cuda_toolchain(self): """Test templates via toolchain component, like setting %(cudaver)s with fosscuda toolchain.""" @@ -1378,7 +1386,7 @@ def test_templating_doc(self): # expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between) temps = [ easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG, - easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3, + list(easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS.keys()) * 3, easyconfig.templates.TEMPLATE_NAMES_CONFIG, easyconfig.templates.TEMPLATE_NAMES_LOWER, easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP,