diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 6fdfcc3841a..c5031254c6f 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -27,8 +27,10 @@ @author: Kenneth Hoste (Ghent University) """ +import os + from easybuild.easyblocks.generic.bundle import Bundle -from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir +from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, PythonPackage, det_pylibdir from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_root @@ -72,6 +74,9 @@ def __init__(self, *args, **kwargs): self.pylibdir = None + # figure out whether this bundle of Python packages is being installed for multiple Python versions + self.multi_python = 'Python' in self.cfg['multi_deps'] + def prepare_step(self, *args, **kwargs): """Prepare for installing bundle of Python packages.""" super(Bundle, self).prepare_step(*args, **kwargs) @@ -90,14 +95,25 @@ def make_module_extra(self, *args, **kwargs): """Extra statements to include in module file: update $PYTHONPATH.""" txt = super(Bundle, self).make_module_extra(*args, **kwargs) - txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir) + # update $EBPYTHONPREFIXES rather than $PYTHONPATH + # if this Python package was installed for multiple Python versions + if self.multi_python: + txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') + else: + txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir) return txt def sanity_check_step(self, *args, **kwargs): """Custom sanity check for bundle of Python package.""" - custom_paths = { - 'files': [], - 'dirs': [self.pylibdir], - } - super(Bundle, self).sanity_check_step(*args, custom_paths=custom_paths, **kwargs) + + # inject directory path that uses %(pyshortver)s template into default value for sanity_check_paths + # this is relevant for installations of Python bundles for multiple Python versions (via multi_deps) + # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved) + if not self.cfg['sanity_check_paths']: + self.cfg['sanity_check_paths'] = { + 'files': [], + 'dirs': [os.path.join('lib', 'python%(pyshortver)s', 'site-packages')], + } + + super(Bundle, self).sanity_check_step(*args, **kwargs) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 82be7ed6c7a..7de21bad227 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -41,7 +41,7 @@ from vsc.utils.missing import nub import easybuild.tools.environment as env -from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES +from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError @@ -234,6 +234,9 @@ def __init__(self, *args, **kwargs): self.log.info("Inherited setting for detection of downloaded dependencies from parent: %s", self.cfg['download_dep_fail']) + # figure out whether this Python package is being installed for multiple Python versions + self.multi_python = 'Python' in self.cfg['multi_deps'] + # determine install command self.use_setup_py = False if self.cfg.get('use_easy_install', False): @@ -538,7 +541,12 @@ def install_step(self): # actually install Python package cmd = self.compose_install_command(self.installdir) - (self.install_cmd_output, _) = run_cmd(cmd, log_all=True, log_ok=True, simple=False) + (out, _) = run_cmd(cmd, log_all=True, log_ok=True, simple=False) + + # keep track of all output from install command, so we can check for auto-downloaded dependencies; + # take into account that install step may be run multiple times + # (for iterated installations over multiply Python versions) + self.install_cmd_output += out # restore PYTHONPATH if it was set if pythonpath is not None: @@ -564,10 +572,6 @@ def sanity_check_step(self, *args, **kwargs): """ Custom sanity check for Python packages """ - if 'exts_filter' not in kwargs: - orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES - exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) - kwargs.update({'exts_filter': exts_filter}) success, fail_msg = True, '' @@ -588,6 +592,29 @@ def sanity_check_step(self, *args, **kwargs): else: self.log.debug("Detection of downloaded dependencies not enabled") + # inject directory path that uses %(pyshortver)s template into default value for sanity_check_paths, + # but only for stand-alone installations, not for extensions; + # this is relevant for installations of Python packages for multiple Python versions (via multi_deps) + # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved) + if not self.is_extension and not self.cfg['sanity_check_paths']: + self.cfg['sanity_check_paths'] = { + 'files': [], + 'dirs': [os.path.join('lib', 'python%(pyshortver)s', 'site-packages')], + } + + # make sure 'exts_filter' is defined, which is used for sanity check + if self.multi_python: + # when installing for multiple Python versions, we must use 'python', not a full-path 'python' command! + if 'exts_filter' not in kwargs: + kwargs.update({'exts_filter': EXTS_FILTER_PYTHON_PACKAGES}) + else: + # 'python' is replaced by full path to active 'python' command + # (which is required especially when installing with system Python) + if 'exts_filter' not in kwargs: + orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES + exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) + kwargs.update({'exts_filter': exts_filter}) + parent_success, parent_fail_msg = super(PythonPackage, self).sanity_check_step(*args, **kwargs) if parent_fail_msg: @@ -620,11 +647,17 @@ def make_module_req_guess(self): def make_module_extra(self, *args, **kwargs): """Add install path to PYTHONPATH""" txt = '' - self.set_pylibdirs() - for path in self.all_pylibdirs: - fullpath = os.path.join(self.installdir, path) - # only extend $PYTHONPATH with existing, non-empty directories - if os.path.exists(fullpath) and os.listdir(fullpath): - txt += self.module_generator.prepend_paths('PYTHONPATH', path) + + # update $EBPYTHONPREFIXES rather than $PYTHONPATH + # if this Python package was installed for multiple Python versions + if self.multi_python: + txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '') + else: + self.set_pylibdirs() + for path in self.all_pylibdirs: + fullpath = os.path.join(self.installdir, path) + # only extend $PYTHONPATH with existing, non-empty directories + if os.path.exists(fullpath) and os.listdir(fullpath): + txt += self.module_generator.prepend_paths('PYTHONPATH', path) return super(PythonPackage, self).make_module_extra(txt, *args, **kwargs)