tweak PythonPackage & PythonBundle to update $EBPYTHONPREFIXES rather than $PYTHONPATH for multi-Python installations + make sanity_check_step aware of multi_deps

easybuild/easyblocks/generic/pythonbundle.py

@@ -27,8 +27,10 @@
 
 @author: Kenneth Hoste (Ghent University)
 """
+import os
+
 from easybuild.easyblocks.generic.bundle import Bundle
-from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir
+from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, PythonPackage, det_pylibdir
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.modules import get_software_root
 
@@ -72,6 +74,9 @@ def __init__(self, *args, **kwargs):
 
         self.pylibdir = None
 
+        # figure out whether this bundle of Python packages is being installed for multiple Python versions
+        self.multi_python = 'Python' in self.cfg['multi_deps']
+
     def prepare_step(self, *args, **kwargs):
         """Prepare for installing bundle of Python packages."""
         super(Bundle, self).prepare_step(*args, **kwargs)
@@ -90,14 +95,25 @@ def make_module_extra(self, *args, **kwargs):
         """Extra statements to include in module file: update $PYTHONPATH."""
         txt = super(Bundle, self).make_module_extra(*args, **kwargs)
 
-        txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir)
+        # update $EBPYTHONPREFIXES rather than $PYTHONPATH
+        # if this Python package was installed for multiple Python versions
+        if self.multi_python:
+            txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '')
+        else:
+            txt += self.module_generator.prepend_paths('PYTHONPATH', self.pylibdir)
 
         return txt
 
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for bundle of Python package."""
-        custom_paths = {
-            'files': [],
-            'dirs': [self.pylibdir],
-        }
-        super(Bundle, self).sanity_check_step(*args, custom_paths=custom_paths, **kwargs)
+
+        # inject directory path that uses %(pyshortver)s template into default value for sanity_check_paths
+        # this is relevant for installations of Python bundles for multiple Python versions (via multi_deps)
+        # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved)
+        if not self.cfg['sanity_check_paths']:
+            self.cfg['sanity_check_paths'] = {
+                'files': [],
+                'dirs': [os.path.join('lib', 'python%(pyshortver)s', 'site-packages')],
+            }
+
+        super(Bundle, self).sanity_check_step(*args, **kwargs)

easybuild/easyblocks/generic/pythonpackage.py

@@ -41,7 +41,7 @@
 from vsc.utils.missing import nub
 
 import easybuild.tools.environment as env
-from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES
+from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
 from easybuild.tools.build_log import EasyBuildError
@@ -234,6 +234,9 @@ def __init__(self, *args, **kwargs):
             self.log.info("Inherited setting for detection of downloaded dependencies from parent: %s",
                           self.cfg['download_dep_fail'])
 
+        # figure out whether this Python package is being installed for multiple Python versions
+        self.multi_python = 'Python' in self.cfg['multi_deps']
+
         # determine install command
         self.use_setup_py = False
         if self.cfg.get('use_easy_install', False):
@@ -538,7 +541,12 @@ def install_step(self):
 
         # actually install Python package
         cmd = self.compose_install_command(self.installdir)
-        (self.install_cmd_output, _) = run_cmd(cmd, log_all=True, log_ok=True, simple=False)
+        (out, _) = run_cmd(cmd, log_all=True, log_ok=True, simple=False)
+
+        # keep track of all output from install command, so we can check for auto-downloaded dependencies;
+        # take into account that install step may be run multiple times
+        # (for iterated installations over multiply Python versions)
+        self.install_cmd_output += out
 
         # restore PYTHONPATH if it was set
         if pythonpath is not None:
@@ -564,10 +572,6 @@ def sanity_check_step(self, *args, **kwargs):
         """
         Custom sanity check for Python packages
         """
-        if 'exts_filter' not in kwargs:
-            orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES
-            exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1])
-            kwargs.update({'exts_filter': exts_filter})
 
         success, fail_msg = True, ''
 
@@ -588,6 +592,29 @@ def sanity_check_step(self, *args, **kwargs):
         else:
             self.log.debug("Detection of downloaded dependencies not enabled")
 
+        # inject directory path that uses %(pyshortver)s template into default value for sanity_check_paths,
+        # but only for stand-alone installations, not for extensions;
+        # this is relevant for installations of Python packages for multiple Python versions (via multi_deps)
+        # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved)
+        if not self.is_extension and not self.cfg['sanity_check_paths']:
+            self.cfg['sanity_check_paths'] = {
+                'files': [],
+                'dirs': [os.path.join('lib', 'python%(pyshortver)s', 'site-packages')],
+            }
+
+        # make sure 'exts_filter' is defined, which is used for sanity check
+        if self.multi_python:
+            # when installing for multiple Python versions, we must use 'python', not a full-path 'python' command!
+            if 'exts_filter' not in kwargs:
+                kwargs.update({'exts_filter': EXTS_FILTER_PYTHON_PACKAGES})
+        else:
+            # 'python' is replaced by full path to active 'python' command
+            # (which is required especially when installing with system Python)
+            if 'exts_filter' not in kwargs:
+                orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES
+                exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1])
+                kwargs.update({'exts_filter': exts_filter})
+
         parent_success, parent_fail_msg = super(PythonPackage, self).sanity_check_step(*args, **kwargs)
 
         if parent_fail_msg:
@@ -620,11 +647,17 @@ def make_module_req_guess(self):
     def make_module_extra(self, *args, **kwargs):
         """Add install path to PYTHONPATH"""
         txt = ''
-        self.set_pylibdirs()
-        for path in self.all_pylibdirs:
-            fullpath = os.path.join(self.installdir, path)
-            # only extend $PYTHONPATH with existing, non-empty directories
-            if os.path.exists(fullpath) and os.listdir(fullpath):
-                txt += self.module_generator.prepend_paths('PYTHONPATH', path)
+
+        # update $EBPYTHONPREFIXES rather than $PYTHONPATH
+        # if this Python package was installed for multiple Python versions
+        if self.multi_python:
+            txt += self.module_generator.prepend_paths(EBPYTHONPREFIXES, '')
+        else:
+            self.set_pylibdirs()
+            for path in self.all_pylibdirs:
+                fullpath = os.path.join(self.installdir, path)
+                # only extend $PYTHONPATH with existing, non-empty directories
+                if os.path.exists(fullpath) and os.listdir(fullpath):
+                    txt += self.module_generator.prepend_paths('PYTHONPATH', path)
 
         return super(PythonPackage, self).make_module_extra(txt, *args, **kwargs)