add support for multi_deps easyconfig parameter (syntactic sugar for iterative list of builddependencies)

easybuild/framework/easyblock.py

@@ -78,8 +78,8 @@
 from easybuild.tools.filetools import move_file, move_logs, read_file, remove_file, rmtree2, verify_checksum, weld_paths
 from easybuild.tools.filetools import write_file
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
-from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTPROC_STEP, PREPARE_STEP
-from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
+from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
+from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
 from easybuild.tools.hooks import load_hooks, run_hook
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
@@ -96,7 +96,7 @@
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
 
 
-MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP]
+MODULE_ONLY_STEPS = [MODULE_STEP, PREPARE_STEP, READY_STEP, POSTITER_STEP, SANITYCHECK_STEP]
 
 # string part of URL for Python packages on PyPI that indicates needs to be rewritten (see derive_alt_pypi_url)
 PYPI_PKG_URL_PATTERN = 'pypi.python.org/packages/source/'
@@ -158,6 +158,10 @@ def __init__(self, ec):
         self.skip = None
         self.module_extra_extensions = ''  # extra stuff for module file required by extensions
 
+        # indicates whether or not this instance represents an extension or not;
+        # may be set to True by ExtensionEasyBlock
+        self.is_extension = False
+
         # easyconfig for this application
         if isinstance(ec, EasyConfig):
             self.cfg = ec
@@ -1484,7 +1488,9 @@ def handle_iterate_opts(self):
         prev_enable_templating = self.cfg.enable_templating
         self.cfg.enable_templating = False
 
-        self.cfg.start_iterating()
+        # start iterative mode (only need to do this once)
+        if self.iter_idx == 0:
+            self.cfg.start_iterating()
 
         # handle configure/build/install options that are specified as lists (+ perhaps builddependencies)
         # set first element to be used, keep track of list in self.iter_opts
@@ -1496,6 +1502,7 @@ def handle_iterate_opts(self):
                 self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt])
 
         if self.iter_opts:
+            print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent)
             self.log.info("Current iteration index: %s", self.iter_idx)
 
         # pop first element from all iterative easyconfig parameters as next value to use
@@ -1512,14 +1519,19 @@ def handle_iterate_opts(self):
         # prepare for next iteration (if any)
         self.iter_idx += 1
 
-    def restore_iterate_opts(self):
+    def post_iter_step(self):
         """Restore options that were iterated over"""
         # disable templating, since we're messing about with values in self.cfg
         prev_enable_templating = self.cfg.enable_templating
         self.cfg.enable_templating = False
 
         for opt in self.iter_opts:
             self.cfg[opt] = self.iter_opts[opt]
+
+            # also need to take into account extensions, since those were iterated over as well
+            for ext in self.ext_instances:
+                ext.cfg[opt] = self.iter_opts[opt]
+
             self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt])
 
         self.cfg.stop_iterating()
@@ -1976,6 +1988,7 @@ def extensions_step(self, fetch=False):
             raise EasyBuildError("Improper default extension class specification, should be list/tuple or string.")
 
         # get class instances for all extensions
+        self.ext_instances = []
         exts_cnt = len(self.exts)
         for idx, ext in enumerate(self.exts):
             self.log.debug("Starting extension %s" % ext['name'])
@@ -2088,6 +2101,7 @@ def post_install_step(self):
         Do some postprocessing
         - run post install commands if any were specified
         """
+
         if self.cfg['postinstallcmds'] is not None:
             # make sure we have a list of commands
             if not isinstance(self.cfg['postinstallcmds'], (list, tuple)):
@@ -2105,9 +2119,50 @@ def sanity_check_step(self, *args, **kwargs):
         """
         if self.dry_run:
             self._sanity_check_step_dry_run(*args, **kwargs)
+
+        # handling of extensions that were installed for multiple dependency versions is done in ExtensionEasyBlock
+        elif self.cfg['multi_deps'] and not self.is_extension:
+            self._sanity_check_step_multi_deps(*args, **kwargs)
+
         else:
             self._sanity_check_step(*args, **kwargs)
 
+    def _sanity_check_step_multi_deps(self, *args, **kwargs):
+        """Perform sanity check for installations that iterate over a list a versions for particular dependencies."""
+
+        # take into account provided list of extra modules (if any)
+        common_extra_modules = kwargs.get('extra_modules') or []
+
+        # if multi_deps was used to do an iterative installation over multiple sets of dependencies,
+        # we need to perform the sanity check for each one of these;
+        # this implies iterating over the list of lists of build dependencies again...
+
+        # get list of (lists of) builddependencies, without templating values
+        builddeps = self.cfg.get_ref('builddependencies')
+
+        # start iterating again;
+        # required to ensure build dependencies are taken into account to resolve templates like %(pyver)s
+        self.cfg.iterating = True
+
+        for iter_deps in self.cfg.get_parsed_multi_deps():
+
+            # need to re-generate template values to get correct values for %(pyver)s and %(pyshortver)s
+            self.cfg['builddependencies'] = iter_deps
+            self.cfg.generate_template_values()
+
+            extra_modules = common_extra_modules + [d['short_mod_name'] for d in iter_deps]
+
+            info_msg = "Running sanity check with extra modules: %s" % ', '.join(extra_modules)
+            trace_msg(info_msg)
+            self.log.info(info_msg)
+
+            kwargs['extra_modules'] = extra_modules
+            self._sanity_check_step(*args, **kwargs)
+
+        # restore list of lists of build dependencies & stop iterating again
+        self.cfg['builddependencies'] = builddeps
+        self.cfg.iterating = False
+
     def sanity_check_rpath(self, rpath_dirs=None):
         """Sanity check binaries/libraries w.r.t. RPATH linking."""
 
@@ -2180,7 +2235,12 @@ def sanity_check_rpath(self, rpath_dirs=None):
         return fails
 
     def _sanity_check_step_common(self, custom_paths, custom_commands):
-        """Determine sanity check paths and commands to use."""
+        """
+        Determine sanity check paths and commands to use.
+
+        :param custom_paths: custom sanity check paths to check existence for
+        :param custom_commands: custom sanity check commands to run
+        """
 
         # supported/required keys in for sanity check paths, along with function used to check the paths
         path_keys_and_check = {
@@ -2248,7 +2308,12 @@ def _sanity_check_step_common(self, custom_paths, custom_commands):
         return paths, path_keys_and_check, commands
 
     def _sanity_check_step_dry_run(self, custom_paths=None, custom_commands=None, **_):
-        """Dry run version of sanity_check_step method."""
+        """
+        Dry run version of sanity_check_step method.
+
+        :param custom_paths: custom sanity check paths to check existence for
+        :param custom_commands: custom sanity check commands to run
+        """
         paths, path_keys_and_check, commands = self._sanity_check_step_common(custom_paths, custom_commands)
 
         for key, (typ, _) in path_keys_and_check.items():
@@ -2299,8 +2364,15 @@ def _sanity_check_step_extensions(self):
             self.sanity_check_fail_msgs.append(overall_fail_msg + ', '.join(x[0] for x in failed_exts))
             self.sanity_check_fail_msgs.extend(x[1] for x in failed_exts)
 
-    def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False):
-        """Real version of sanity_check_step method."""
+    def _sanity_check_step(self, custom_paths=None, custom_commands=None, extension=False, extra_modules=None):
+        """
+        Real version of sanity_check_step method.
+
+        :param custom_paths: custom sanity check paths to check existence for
+        :param custom_commands: custom sanity check commands to run
+        :param extension: indicates whether or not sanity check is run for an extension
+        :param extra_modules: extra modules to load before running sanity check commands
+        """
         paths, path_keys_and_check, commands = self._sanity_check_step_common(custom_paths, custom_commands)
 
         # helper function to sanity check (alternatives for) one particular path
@@ -2355,6 +2427,7 @@ def xs2str(xs):
                 trace_msg("%s %s found: %s" % (typ, xs2str(xs), ('FAILED', 'OK')[found]))
 
         fake_mod_data = None
+
         # only load fake module for non-extensions, and not during dry run
         if not (extension or self.dry_run):
             try:
@@ -2365,6 +2438,11 @@ def xs2str(xs):
                 self.sanity_check_fail_msgs.append("loading fake module failed: %s" % err)
                 self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1])
 
+            # also load specificed additional modules, if any
+            if extra_modules:
+                self.log.info("Loading extra modules for sanity check: %s", ', '.join(extra_modules))
+                self.modules_tool.load(extra_modules)
+
         # chdir to installdir (better environment for running tests)
         if os.path.isdir(self.installdir):
             change_dir(self.installdir)
@@ -2449,8 +2527,6 @@ def cleanup_step(self):
 
         env.restore_env_vars(self.cfg['unwanted_env_vars'])
 
-        self.restore_iterate_opts()
-
     def invalidate_module_caches(self, modpath):
         """Helper method to invalidate module caches for specified module path."""
         # invalidate relevant 'module avail'/'module show' cache entries
@@ -2729,6 +2805,7 @@ def install_step_spec(initial):
         configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True)
         build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True)
         test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True)
+        extensions_step_spec = (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False)
 
         # part 1: pre-iteration + first iteration
         steps_part1 = [
@@ -2741,6 +2818,7 @@ def install_step_spec(initial):
             build_step_spec,
             test_step_spec,
             install_step_spec(True),
+            extensions_step_spec,
         ]
         # part 2: iterated part, from 2nd iteration onwards
         # repeat core procedure again depending on specified iteration count
@@ -2754,10 +2832,11 @@ def install_step_spec(initial):
             build_step_spec,
             test_step_spec,
             install_step_spec(False),
+            extensions_step_spec,
         ] * (iteration_count - 1)
         # part 3: post-iteration part
         steps_part3 = [
-            (EXTENSIONS_STEP, 'taking care of extensions', [lambda x: x.extensions_step], False),
+            (POSTITER_STEP, 'restore after iterating', [lambda x: x.post_iter_step], False),
             (POSTPROC_STEP, 'postprocessing', [lambda x: x.post_install_step], True),
             (SANITYCHECK_STEP, 'sanity checking', [lambda x: x.sanity_check_step], True),
             (CLEANUP_STEP, 'cleaning up', [lambda x: x.cleanup_step], False),

easybuild/framework/easyconfig/default.py

@@ -146,6 +146,7 @@
     'builddependencies': [[], "List of build dependencies", DEPENDENCIES],
     'dependencies': [[], "List of dependencies", DEPENDENCIES],
     'hiddendependencies': [[], "List of dependencies available as hidden modules", DEPENDENCIES],
+    'multi_deps': [{}, "Dict of lists of dependency versions over which to iterate", DEPENDENCIES],
     'osdependencies': [[], "OS dependencies that should be present on the system", DEPENDENCIES],
 
     # LICENSE easyconfig parameters

easybuild/framework/easyconfig/easyconfig.py

@@ -385,6 +385,10 @@ def __init__(self, path, extra_options=None, build_specs=None, validate=True, hi
 
         self.external_modules_metadata = build_option('external_modules_metadata')
 
+        # list of all options to iterate over
+        self.iterate_options = []
+        self.iterating = False
+
         # parse easyconfig file
         self.build_specs = build_specs
         self.parse()
@@ -527,26 +531,23 @@ def parse(self):
             else:
                 self.log.debug("Ignoring unknown easyconfig parameter %s (value: %s)" % (key, local_vars[key]))
 
-        # trigger parse hook
         # templating is disabled when parse_hook is called to allow for easy updating of mutable easyconfig parameters
         # (see also comment in resolve_template)
-        hooks = load_hooks(build_option('hooks'))
         prev_enable_templating = self.enable_templating
         self.enable_templating = False
 
+        # if any lists of dependency versions are specified over which we should iterate,
+        # deal with them now, before calling parse hook, parsing of dependencies & iterative easyconfig parameters...
+        self.handle_multi_deps()
+
         parse_hook_msg = None
         if self.path:
             parse_hook_msg = "Running %s hook for %s..." % (PARSE, os.path.basename(self.path))
 
+        # trigger parse hook
+        hooks = load_hooks(build_option('hooks'))
         run_hook(PARSE, hooks, args=[self], msg=parse_hook_msg)
 
-        # create a list of all options that are actually going to be iterated over
-        # builddependencies are always a list, need to look deeper down below
-
-        # list of all options to iterate over
-        self.iterate_options = []
-        self.iterating = False
-
         # parse dependency specifications
         # it's important that templating is still disabled at this stage!
         self.log.info("Parsing dependency specifications...")
@@ -731,7 +732,7 @@ def filter_hidden_deps(self):
         deps = dict([(key, self.get_ref(key)) for key in ['dependencies', 'builddependencies', 'hiddendependencies']])
 
         if 'builddependencies' in self.iterate_options:
-            deplists = deps['builddependencies']
+            deplists = copy.deepcopy(deps['builddependencies'])
         else:
             deplists = [deps['builddependencies']]
 
@@ -1032,6 +1033,69 @@ def handle_external_module_metadata(self, dep_name):
 
         return dependency
 
+    def handle_multi_deps(self):
+        """
+        Handle lists of dependency versions of which we should iterate specified in 'multi_deps' easyconfig parameter.
+
+        This is basically just syntactic sugar to prevent having to specify a list of lists in 'builddependencies'.
+        """
+
+        multi_deps = self['multi_deps']
+        if multi_deps:
+
+            # first, make sure all lists have same length, otherwise we're dealing with invalid input...
+            multi_dep_cnts = nub([len(dep_vers) for dep_vers in multi_deps.values()])
+            if len(multi_dep_cnts) == 1:
+                multi_dep_cnt = multi_dep_cnts[0]
+            else:
+                raise EasyBuildError("Not all the dependencies listed in multi_deps have the same number of versions!")
+
+            self.log.info("Found %d lists of %d dependency versions to iterate over", len(multi_deps), multi_dep_cnt)
+
+            # make sure that build dependencies is not a list of lists to iterate over already...
+            if self['builddependencies'] and all(isinstance(bd, list) for bd in self['builddependencies']):
+                raise EasyBuildError("Can't combine multi_deps with builddependencies specified as list of lists")
+
+            # now make builddependencies a list of lists to iterate over
+            builddeps = self['builddependencies']
+            self['builddependencies'] = []
+
+            keys = sorted(multi_deps.keys())
+            for idx in range(multi_dep_cnt):
+                self['builddependencies'].append(builddeps + [(key, multi_deps[key][idx]) for key in keys])
+
+            self.log.info("Original list of build dependencies: %s", builddeps)
+            self.log.info("List of lists of build dependencies to iterate over: %s", self['builddependencies'])
+
+    def get_parsed_multi_deps(self):
+        """Get list of lists of parsed dependencies that correspond with entries in multi_deps easyconfig parameter."""
+
+        multi_deps = []
+
+        builddeps = self['builddependencies']
+
+        # all multi_deps entries should be listed in builddependencies (if not, something is very wrong)
+        if isinstance(builddeps, list) and all(isinstance(x, list) for x in builddeps):
+
+            for iter_id in range(len(builddeps)):
+
+                # only build dependencies that correspond to multi_deps entries should be loaded as extra modules
+                # (other build dependencies should not be required to make sanity check pass for this iteration)
+                iter_deps = []
+                for key in self['multi_deps']:
+                    hits = [d for d in builddeps[iter_id] if d['name'] == key]
+                    if len(hits) == 1:
+                        iter_deps.append(hits[0])
+                    else:
+                        raise EasyBuildError("Failed to isolate %s dep during iter #%d: %s", key, iter_id, hits)
+
+                multi_deps.append(iter_deps)
+        else:
+            error_msg = "builddependencies should be a list of lists when calling get_multi_deps(), but it's not: %s"
+            raise EasyBuildError(error_msg, builddeps)
+
+        return multi_deps
+
     # private method
     def _parse_dependency(self, dep, hidden=False, build_only=False):
         """

easybuild/framework/easyconfig/format/format.py

@@ -54,7 +54,7 @@
 
 # values for these keys will not be templated in dump()
 EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'exts_list', 'homepage', 'name', 'toolchain',
-                                   'version'] + DEPENDENCY_PARAMETERS
+                                   'version', 'multi_deps'] + DEPENDENCY_PARAMETERS
 
 # ordered groups of keys to obtain a nice looking easyconfig file
 GROUPED_PARAMS = [
@@ -63,7 +63,7 @@
     ['homepage', 'description'],
     ['toolchain', 'toolchainopts'],
     ['source_urls', 'sources', 'patches', 'checksums'],
-    DEPENDENCY_PARAMETERS,
+    DEPENDENCY_PARAMETERS + ['multi_deps'],
     ['osdependencies'],
     ['preconfigopts', 'configopts'],
     ['prebuildopts', 'buildopts'],