allow specifying location for RPATH wrapper scripts via rpath_wrappers_dir + also pass rpath_include_dirs when preparing build environment for extensions

easybuild/framework/easyblock.py

@@ -238,6 +238,9 @@ def __init__(self, ec, logfile=None):
         # list of locations to include in RPATH used by toolchain
         self.rpath_include_dirs = []
 
+        # directory to export RPATH wrappers to
+        self.rpath_wrappers_dir = None
+
         # logging
         self.log = None
         self.logfile = logfile
@@ -2103,7 +2106,9 @@ def install_extensions_sequential(self, install=True):
                 # don't reload modules for toolchain, there is no need since they will be loaded already;
                 # the (fake) module for the parent software gets loaded before installing extensions
                 ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
-                                      rpath_filter_dirs=self.rpath_filter_dirs)
+                                      rpath_filter_dirs=self.rpath_filter_dirs,
+                                      rpath_include_dirs=self.rpath_include_dirs,
+                                      rpath_wrappers_dir=self.rpath_wrappers_dir)
 
             # actual installation of the extension
             if install:
@@ -2265,7 +2270,9 @@ def update_exts_progress_bar_helper(running_exts, progress_size):
                     # don't reload modules for toolchain, there is no need since they will be loaded already;
                     # the (fake) module for the parent software gets loaded before installing extensions
                     ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False,
-                                          rpath_filter_dirs=self.rpath_filter_dirs)
+                                          rpath_filter_dirs=self.rpath_filter_dirs,
+                                          rpath_include_dirs=self.rpath_include_dirs,
+                                          rpath_wrappers_dir=self.rpath_wrappers_dir)
                     if install:
                         ext.install_extension_substep("pre_install_extension")
                         ext.async_cmd_task = ext.install_extension_substep("install_extension_async", thread_pool)
@@ -2894,6 +2901,14 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
             '$ORIGIN/../lib64',
         ])
 
+        # Location to store RPATH wrappers
+        if self.rpath_wrappers_dir is not None:
+            # Verify the path given is absolute
+            if os.path.isabs(self.rpath_wrappers_dir):
+                _log.info(f"Using {self.rpath_wrappers_dir} to store/use RPATH wrappers")
+            else:
+                raise EasyBuildError(f"Path used for rpath_wrappers_dir is not an absolute path: {path}")
+
         if self.iter_idx > 0:
             # reset toolchain for iterative runs before preparing it again
             self.toolchain.reset()
@@ -2909,9 +2924,11 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
                 self.modules_tool.prepend_module_path(full_mod_path)
 
         # prepare toolchain: load toolchain module and dependencies, set up build environment
-        self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent,
-                               loadmod=load_tc_deps_modules, rpath_filter_dirs=self.rpath_filter_dirs,
-                               rpath_include_dirs=self.rpath_include_dirs)
+        self.toolchain.prepare(onlymod=self.cfg['onlytcmod'], deps=self.cfg.dependencies(),
+                               silent=self.silent, loadmod=load_tc_deps_modules,
+                               rpath_filter_dirs=self.rpath_filter_dirs,
+                               rpath_include_dirs=self.rpath_include_dirs,
+                               rpath_wrappers_dir=self.rpath_wrappers_dir)
 
         # keep track of environment variables that were tweaked and need to be restored after environment got reset
         # $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths...

easybuild/scripts/rpath_wrapper_template.sh.in

@@ -42,6 +42,7 @@ function log {
 
 # command name
 CMD=`basename $0`
+TOPDIR=`dirname $0`
 
 log "found CMD: $CMD | original command: %(orig_cmd)s | orig args: '$(echo \"$@\")'"
 

easybuild/tools/toolchain/toolchain.py

@@ -62,7 +62,7 @@
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_warning
 from easybuild.tools.config import build_option, install_path
 from easybuild.tools.environment import setvar
-from easybuild.tools.filetools import adjust_permissions, find_eb_script, read_file, which, write_file
+from easybuild.tools.filetools import adjust_permissions, copy_file, find_eb_script, mkdir, read_file, which, write_file
 from easybuild.tools.module_generator import dependencies_for
 from easybuild.tools.modules import get_software_root, get_software_root_env_var_name
 from easybuild.tools.modules import get_software_version, get_software_version_env_var_name
@@ -839,7 +839,7 @@ def reset(self):
         self.variables_init()
 
     def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
-                rpath_filter_dirs=None, rpath_include_dirs=None):
+                rpath_filter_dirs=None, rpath_include_dirs=None, rpath_wrappers_dir=None):
         """
         Prepare a set of environment parameters based on name/version of toolchain
         - load modules for toolchain and dependencies
@@ -853,6 +853,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
         :param loadmod: whether or not to (re)load the toolchain module, and the modules for the dependencies
         :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
         :param rpath_include_dirs: extra directories to include in RPATH
+        :param rpath_wrappers_dir: directory in which to create RPATH wrappers
         """
 
         # take into account --sysroot configuration setting
@@ -906,7 +907,11 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
 
         if build_option('rpath'):
             if self.options.get('rpath', True):
-                self.prepare_rpath_wrappers(rpath_filter_dirs, rpath_include_dirs)
+                self.prepare_rpath_wrappers(
+                    rpath_filter_dirs=rpath_filter_dirs,
+                    rpath_include_dirs=rpath_include_dirs,
+                    rpath_wrappers_dir=rpath_wrappers_dir
+                    )
                 self.use_rpath = True
             else:
                 self.log.info("Not putting RPATH wrappers in place, disabled via 'rpath' toolchain option")
@@ -975,11 +980,13 @@ def is_rpath_wrapper(path):
         # need to use binary mode to read the file, since it may be an actual compiler command (which is a binary file)
         return b'rpath_args.py $CMD' in read_file(path, mode='rb')
 
-    def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None):
+    def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None, rpath_wrappers_dir=None):
         """
         Put RPATH wrapper script in place for compiler and linker commands
 
         :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
+        :param rpath_include_dirs: extra directories to include in RPATH
+        :param rpath_wrappers_dir: directory in which to create RPATH wrappers (tmpdir is created if None)
         """
         if get_os_type() == LINUX:
             self.log.info("Putting RPATH wrappers in place...")
@@ -989,6 +996,11 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
         if rpath_filter_dirs is None:
             rpath_filter_dirs = []
 
+        # only enable logging by RPATH wrapper scripts in debug mode
+        enable_wrapper_log = build_option('debug')
+
+        copy_rpath_args_py = False
+
         # always include filter for 'stubs' library directory,
         # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
         # (since CUDA 11.something the stubs are in $EBROOTCUDA/stubs/lib64)
@@ -997,13 +1009,35 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
             if lib_stubs_pattern not in rpath_filter_dirs:
                 rpath_filter_dirs.append(lib_stubs_pattern)
 
-        # directory where all wrappers will be placed
-        wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
+        # directory where all RPATH wrapper script will be placed;
+        if rpath_wrappers_dir is None:
+            wrappers_dir = tempfile.mkdtemp()
+        else:
+            wrappers_dir = rpath_wrappers_dir
+            # disable logging in RPATH wrapper scripts when they may be exported for use outside of EasyBuild
+            enable_wrapper_log = False
+            # copy rpath_args.py script to sit alongside RPATH wrapper scripts
+            copy_rpath_args_py = True
+
+        # it's important to honor RPATH_WRAPPERS_SUBDIR, see is_rpath_wrapper method
+        wrappers_dir = os.path.join(wrappers_dir, RPATH_WRAPPERS_SUBDIR)
+        mkdir(wrappers_dir, parents=True)
 
         # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
         c_comps, fortran_comps = self.compilers()
 
         rpath_args_py = find_eb_script('rpath_args.py')
+
+        # copy rpath_args.py script along RPATH wrappers, if desired
+        if copy_rpath_args_py:
+            copy_file(rpath_args_py, wrappers_dir)
+            # use path for %(rpath_args)s template value relative to location of the RPATH wrapper script,
+            # to avoid that the RPATH wrapper scripts rely on a script that's located elsewhere;
+            # that's mostly important when RPATH wrapper scripts are retained to be used outside of EasyBuild;
+            # we assume that each RPATH wrapper script is created in a separate subdirectory (see wrapper_dir below);
+            # ${TOPDIR} is defined in template for RPATH wrapper scripts, refers to parent dir of RPATH wrapper script
+            rpath_args_py = os.path.join('${TOPDIR}', '..', os.path.basename(rpath_args_py))
+
         rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
 
         # figure out list of patterns to use in rpath filter
@@ -1042,11 +1076,11 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
 
                 # make *very* sure we don't wrap around ourselves and create a fork bomb...
                 if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
-                    raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd)
+                    raise EasyBuildError("Refusing to create a fork bomb, which(%s) == %s", cmd, orig_cmd)
 
                 # enable debug mode in wrapper script by specifying location for log file
-                if build_option('debug'):
-                    rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
+                if enable_wrapper_log:
+                    rpath_wrapper_log = os.path.join(tempfile.gettempdir(), f'rpath_wrapper_{cmd}.log')
                 else:
                     rpath_wrapper_log = '/dev/null'
 
@@ -1060,7 +1094,15 @@ def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None
                     'rpath_wrapper_log': rpath_wrapper_log,
                     'wrapper_dir': wrapper_dir,
                 }
-                write_file(cmd_wrapper, cmd_wrapper_txt)
+
+                # it may be the case that the wrapper already exists if the user provides a fixed location to store
+                # the RPATH wrappers, in this case the wrappers will be overwritten as they do not yet appear in the
+                # PATH (`which(cmd)` does not "see" them). Warn that they will be overwritten.
+                if os.path.exists(cmd_wrapper):
+                    _log.warning(f"Overwriting existing RPATH wrapper {cmd_wrapper}")
+                    write_file(cmd_wrapper, cmd_wrapper_txt, always_overwrite=True)
+                else:
+                    write_file(cmd_wrapper, cmd_wrapper_txt)
                 adjust_permissions(cmd_wrapper, stat.S_IXUSR)
 
                 # prepend location to this wrapper to $PATH

test/framework/toolchain.py

@@ -54,7 +54,7 @@
 from easybuild.tools.run import run_shell_cmd
 from easybuild.tools.systemtools import get_shared_lib_ext
 from easybuild.tools.toolchain.mpi import get_mpi_cmd_template
-from easybuild.tools.toolchain.toolchain import env_vars_external_module
+from easybuild.tools.toolchain.toolchain import env_vars_external_module, RPATH_WRAPPERS_SUBDIR
 from easybuild.tools.toolchain.utilities import get_toolchain, search_toolchain
 from easybuild.toolchains.compiler.clang import Clang
 
@@ -3137,6 +3137,33 @@ def test_toolchain_prepare_rpath(self):
         self.assertTrue(os.path.samefile(res[1], fake_gxx))
         self.assertFalse(any(os.path.samefile(x, fake_gxx) for x in res[2:]))
 
+    def test_toolchain_prepare_rpath_external(self):
+        """Test toolchain.prepare under --rpath with rpath_wrappers_dir argument"""
+
+        # put fake 'g++' command in place that just echos its arguments
+        fake_gxx = os.path.join(self.test_prefix, 'fake', 'g++')
+        write_file(fake_gxx, '#!/bin/bash\necho "$@"')
+        adjust_permissions(fake_gxx, stat.S_IXUSR)
+        os.environ['PATH'] = '%s:%s' % (os.path.join(self.test_prefix, 'fake'), os.getenv('PATH', ''))
+
+        # export the wrappers to a target location
+        target_wrapper_dir = os.path.abspath(os.path.join(self.test_prefix, 'target'))
+        # enable --rpath for a toolchain so we test against it
+        init_config(build_options={'rpath': True, 'silent': True})
+        tc = self.get_toolchain('gompi', version='2018a')
+        tc.set_options({'rpath': True})
+        # allow the underlying toolchain to be in a prepared state (which may include rpath wrapping)
+        with self.mocked_stdout_stderr():
+            tc.prepare(rpath_wrappers_dir=target_wrapper_dir)
+
+        # check that wrapper was created
+        target_wrapper = os.path.join(target_wrapper_dir, RPATH_WRAPPERS_SUBDIR, 'gxx_wrapper', 'g++')
+        self.assertTrue(os.path.exists(target_wrapper))
+        # Make sure it is a wrapper
+        self.assertTrue(b'rpath_args.py $CMD' in read_file(target_wrapper, mode='rb'))
+        # Make sure it wraps our fake 'g++'
+        self.assertTrue(fake_gxx.encode(encoding="utf-8") in read_file(target_wrapper, mode='rb'))
+
     def test_prepare_openmpi_tmpdir(self):
         """Test handling of long $TMPDIR path for OpenMPI 2.x"""