introduce get_parallel_ec_param_value function to determine value for 'parallel' easyconfig parameter for easyblock and extensions

Author: boegel

easybuild/framework/easyblock.py

@@ -55,7 +55,8 @@
 from easybuild.base import fancylogger
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
 from easybuild.framework.easyconfig.easyconfig import ITERATE_OPTIONS, EasyConfig, ActiveMNS, get_easyblock_class
-from easybuild.framework.easyconfig.easyconfig import get_module_path, letter_dir_for, resolve_template
+from easybuild.framework.easyconfig.easyconfig import get_module_path, get_parallel_ec_param_value
+from easybuild.framework.easyconfig.easyconfig import letter_dir_for, resolve_template
 from easybuild.framework.easyconfig.format.format import SANITY_CHECK_PATHS_DIRS, SANITY_CHECK_PATHS_FILES
 from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
 from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH
@@ -92,7 +93,7 @@
 from easybuild.tools.package.utilities import package
 from easybuild.tools.py2vs3 import extract_method_name, string_type
 from easybuild.tools.repository.repository import init_repository
-from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_shared_lib_ext, use_group
+from easybuild.tools.systemtools import check_linked_shared_libs, get_shared_lib_ext, use_group
 from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str
 from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg
 from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION
@@ -212,6 +213,8 @@ def __init__(self, ec):
         self.postmsg = ''  # allow a post message to be set, which can be shown as last output
         self.current_step = None
 
+        self.orig_parallel = None
+
         # list of loaded modules
         self.loaded_modules = []
 
@@ -1789,20 +1792,13 @@ def det_iter_cnt(self):
 
     def set_parallel(self):
         """Set 'parallel' easyconfig parameter to determine how many cores can/should be used for parallel builds."""
-        # set level of parallelism for build
-        par = build_option('parallel')
-        cfg_par = self.cfg['parallel']
-        if cfg_par is None:
-            self.log.debug("Desired parallelism specified via 'parallel' build option: %s", par)
-        elif par is None:
-            par = cfg_par
-            self.log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
-        else:
-            par = min(int(par), int(cfg_par))
-            self.log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
+        # keep track of original value for 'parallel', so it can be restored before determining level
+        # of parallelism that can be used for extensions
+        self.orig_parallel = self.cfg['parallel']
+
+        par = get_parallel_ec_param_value(self.cfg, self.log)
 
-        par = det_parallelism(par, maxpar=self.cfg['maxparallel'])
-        self.log.info("Setting parallelism: %s" % par)
+        self.log.info("Setting parallelism: %s", par)
         self.cfg['parallel'] = par
 
     def remove_module_file(self):

easybuild/framework/easyconfig/easyconfig.py

@@ -73,7 +73,7 @@
 from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name
 from easybuild.tools.modules import modules_tool
 from easybuild.tools.py2vs3 import OrderedDict, create_base_metaclass, string_type
-from easybuild.tools.systemtools import check_os_dependency, pick_dep_version
+from easybuild.tools.systemtools import check_os_dependency, det_parallelism, pick_dep_version
 from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain
 from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA
 from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
@@ -1830,6 +1830,30 @@ def get_cuda_cc_template_value(self, key):
             raise EasyBuildError(error_msg, key)
 
 
+def get_parallel_ec_param_value(cfg, log):
+    """
+    Get value for 'parallel' easyconfig parameter for given EasyConfig instance.
+    Takes into account:
+    * --parallel EasyBuild configuration option (if defined)
+    * 'parallel' easyconfig parameter (if defined)
+    * 'maxparallel' easyconfig parameter (if defined)
+    * number of available cores (incl. affinity of active EasyBuild session)
+    """
+    # set level of parallelism for build
+    par = build_option('parallel')
+    cfg_par = cfg['parallel']
+    if cfg_par is None:
+        log.debug("Desired parallelism specified via 'parallel' build option: %s", par)
+    elif par is None:
+        par = cfg_par
+        log.debug("Desired parallelism specified via 'parallel' easyconfig parameter: %s", par)
+    else:
+        par = min(int(par), int(cfg_par))
+        log.debug("Desired parallelism: minimum of 'parallel' build option/easyconfig parameter: %s", par)
+
+    return det_parallelism(par, maxpar=cfg['maxparallel'])
+
+
 def det_installversion(version, toolchain_name, toolchain_version, prefix, suffix):
     """Deprecated 'det_installversion' function, to determine exact install version, based on supplied parameters."""
     old_fn = 'framework.easyconfig.easyconfig.det_installversion'

easybuild/framework/extension.py

@@ -36,7 +36,7 @@
 import copy
 import os
 
-from easybuild.framework.easyconfig.easyconfig import resolve_template
+from easybuild.framework.easyconfig.easyconfig import get_parallel_ec_param_value, resolve_template
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
 from easybuild.tools.build_log import EasyBuildError, raise_nosupport
 from easybuild.tools.filetools import change_dir
@@ -138,9 +138,11 @@ def __init__(self, mself, ext, extra_params=None):
                 self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s",
                                key, name, version, value)
 
-        if self.cfg['maxparallel'] and self.cfg['parallel'] > self.cfg['maxparallel']:
-            self.cfg['parallel'] = self.cfg['maxparallel']
-            self.log.info("Setting parallelism to %d for extension %s", self.cfg['parallel'], name)
+        self.cfg['parallel'] = self.master.orig_parallel
+        par = get_parallel_ec_param_value(self.cfg, self.log)
+        self.log.info("Setting parallelism: %s", par)
+        self.log.info("Setting parallelism to %d for extension %s", par, name)
+        self.cfg['parallel'] = par
 
         self.sanity_check_fail_msgs = []
 

test/framework/easyblock.py

@@ -1026,6 +1026,7 @@ def test_init_extensions(self):
         test_ec = os.path.join(self.test_prefix, 'test.eb')
         test_ec_txt = toy_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'easyblock': 'DummyExtension',")
         test_ec_txt = test_ec_txt.replace("('barbar', '0.0', {", "('barbar', '0.0', {'maxparallel': 3,")
+        test_ec_txt = test_ec_txt.replace("('bar', '0.0', {", "('bar', '0.0', {'maxparallel': 1,")
         test_ec_txt += "\nparallel = 5"
         write_file(test_ec, test_ec_txt)
         ec = process_easyconfig(test_ec)[0]
@@ -1048,9 +1049,13 @@ def test_init_extensions(self):
         # default parallel is inherited from parent
         self.assertEqual(eb.cfg['parallel'], 5)
         self.assertEqual(eb.ext_instances[0].cfg['parallel'], 5)
-        self.assertEqual(eb.ext_instances[1].cfg['parallel'], 5)
         self.assertEqual(eb.ext_instances[3].cfg['parallel'], 5)
 
+        bar_ext = eb.ext_instances[1]
+        self.assertEqual(bar_ext.name, 'bar')
+        # parallel should be set to 1 for barbar, due to maxparallel
+        self.assertEqual(bar_ext.cfg['parallel'], 1)
+
         barbar_ext = eb.ext_instances[2]
         self.assertEqual(barbar_ext.name, 'barbar')
         # parallel should be set to 3 for barbar, due to maxparallel