diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1eb610b2c8..b44ac74f05 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -109,7 +109,7 @@ from easybuild.tools.repository.repository import init_repository from easybuild.tools.systemtools import check_linked_shared_libs, det_parallelism, get_linked_libs_raw from easybuild.tools.systemtools import get_shared_lib_ext, pick_system_specific_value, use_group -from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, nub, quote_str +from easybuild.tools.utilities import INDENT_4SPACES, get_class_for, quote_str from easybuild.tools.utilities import remove_unwanted_chars, time2str, trace_msg from easybuild.tools.version import this_is_easybuild, VERBOSE_VERSION, VERSION @@ -126,17 +126,6 @@ _log = fancylogger.getLogger('easyblock') -class LibSymlink(Enum): - """ - Possible states for symlinking of lib/lib64 subdirectories: - - UNKNOWN: has not been determined yet - - LIB_TO_LIB64: 'lib' is a symlink to 'lib64' - - LIB64_TO_LIB: 'lib64' is a symlink to 'lib' - - NEITHER: neither 'lib' is a symlink to 'lib64', nor 'lib64' is a symlink to 'lib' - - """ - LIB_TO_LIB64, LIB64_TO_LIB, NEITHER = range(3) - - class EasyBlock(object): """Generic support for building and installing software, base class for actual easyblocks.""" @@ -225,9 +214,6 @@ def __init__(self, ec, logfile=None): # determine install subdirectory, based on module name self.install_subdir = None - # track status of symlink between library directories - self._install_lib_symlink = None - # indicates whether build should be performed in installation dir self.build_in_installdir = self.cfg['buildininstalldir'] @@ -311,13 +297,6 @@ def __init__(self, ec, logfile=None): self.log.info("Init completed for application name %s version %s" % (self.name, self.version)) - @property - def install_lib_symlink(self): - """Return symlink state of lib/lib64 folders""" - if self._install_lib_symlink is None: - self.check_install_lib_symlink() - return self._install_lib_symlink - def post_init(self): """ Run post-initialization tasks. @@ -1694,15 +1673,47 @@ def make_module_req(self): for env_var, search_paths in env_var_requirements.items(): if self.dry_run: self.dry_run_msg(f" ${env_var}:{', '.join(search_paths)}") - # Don't expand globs or do any filtering for dry run - mod_req_paths = search_paths + # Don't expand globs or do any filtering for dry run, + # convert to regular list that can be subscripted below + mod_req_paths = list(search_paths) else: mod_req_paths = [] for path in search_paths: mod_req_paths.extend(self.expand_module_search_path(path, path_type=search_paths.type)) if mod_req_paths: - mod_req_paths = nub(mod_req_paths) # remove duplicates + # find duplicate paths (taking into account possible symlinks) + dup_paths = [] + # always retain first entry + retained_paths = [mod_req_paths[0]] + full_retained_paths = [os.path.join(self.installdir, retained_paths[0])] + + for path in mod_req_paths[1:]: + full_path = os.path.join(self.installdir, path) + # retain all paths in dry run mode (since then paths may not exist) + if self.dry_run: + retained_paths.append(path) + elif os.path.exists(full_path) and any(os.path.samefile(full_path, p) for p in full_retained_paths): + dup_paths.append(path) + else: + retained_paths.append(path) + full_retained_paths = [os.path.join(self.installdir, p) for p in retained_paths] + + if dup_paths: + self.log.info(f"Filtering out duplicate paths for ${env_var}: {dup_paths}") + mod_req_paths = retained_paths + self.log.info(f"Retained paths for ${env_var}: {mod_req_paths}") + else: + self.log.info(f"No duplicate paths found for ${env_var}: {mod_req_paths}") + + # for $CMAKE_LIBRARY_PATH, only retain 'lib64' if it's standalone (*not* a symlink to 'lib') + if env_var == 'CMAKE_LIBRARY_PATH' and 'lib64' in mod_req_paths: + full_lib = os.path.join(self.installdir, 'lib') + full_lib64 = os.path.join(self.installdir, 'lib64') + if os.path.exists(full_lib64) and os.path.exists(full_lib): + if os.path.samefile(full_lib64, full_lib): + mod_req_paths.remove('lib64') + mod_lines.append(self.module_generator.prepend_paths(env_var, mod_req_paths)) if self.dry_run: @@ -1732,15 +1743,6 @@ def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WI tentative_path = os.path.relpath(abs_path, start=self.installdir) tentative_path = '' if tentative_path == '.' else tentative_path # use empty string instead of dot - # avoid duplicate entries between symlinked library dirs - tent_path_sep = tentative_path + os.path.sep - if self.install_lib_symlink == LibSymlink.LIB64_TO_LIB and tent_path_sep.startswith('lib64' + os.path.sep): - self.log.debug("Discarded search path to symlinked lib64 directory: %s", tentative_path) - continue - if self.install_lib_symlink == LibSymlink.LIB_TO_LIB64 and tent_path_sep.startswith('lib' + os.path.sep): - self.log.debug("Discarded search path to symlinked lib directory: %s", tentative_path) - continue - check_dir_files = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.PATH_WITH_TOP_FILES) if os.path.isdir(abs_path) and check_dir_files: # only retain paths to directories that contain at least one file @@ -1753,18 +1755,6 @@ def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WI return retained_search_paths - def check_install_lib_symlink(self): - """Update the symlink state between library directories in installation prefix""" - lib_dir = os.path.join(self.installdir, 'lib') - lib64_dir = os.path.join(self.installdir, 'lib64') - - self._install_lib_symlink = LibSymlink.NEITHER - if os.path.exists(lib_dir) and os.path.exists(lib64_dir): - if os.path.islink(lib_dir) and os.path.samefile(lib_dir, lib64_dir): - self._install_lib_symlink = LibSymlink.LIB_TO_LIB64 - elif os.path.islink(lib64_dir) and os.path.samefile(lib_dir, lib64_dir): - self._install_lib_symlink = LibSymlink.LIB64_TO_LIB - def make_module_req_guess(self): """ A dictionary of common search path variables to be loaded by environment modules @@ -3218,9 +3208,6 @@ def post_install_step(self): # create *relative* 'lib' symlink to 'lib64'; symlink('lib64', lib_dir, use_abspath_source=False) - # refresh symlink state in install_lib_symlink class variable - self.check_install_lib_symlink() - self.run_post_install_commands() self.apply_post_install_patches() self.print_post_install_messages() diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 59c061d78e..88a64fc0ab 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -42,7 +42,7 @@ import easybuild.tools.systemtools as st from easybuild.base import fancylogger -from easybuild.framework.easyblock import EasyBlock, LibSymlink, get_easyblock_instance +from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.tools import avail_easyblocks, process_easyconfig @@ -52,7 +52,7 @@ from easybuild.tools.config import get_module_syntax, update_build_option from easybuild.tools.environment import modify_env from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file -from easybuild.tools.filetools import verify_checksum, write_file +from easybuild.tools.filetools import symlink, verify_checksum, write_file from easybuild.tools.module_generator import module_generator from easybuild.tools.modules import EnvironmentModules, Lmod, ModEnvVarType, reset_module_caches from easybuild.tools.version import get_git_revision, this_is_easybuild @@ -439,7 +439,6 @@ def test_make_module_req(self): for path in ('bin', ('bin', 'testdir'), 'sbin', 'share', ('share', 'man'), 'lib', 'lib64'): path_components = (path, ) if isinstance(path, str) else path os.mkdir(os.path.join(eb.installdir, *path_components)) - eb.check_install_lib_symlink() write_file(os.path.join(eb.installdir, 'foo.jar'), 'foo.jar') write_file(os.path.join(eb.installdir, 'bla.jar'), 'bla.jar') @@ -505,7 +504,6 @@ def test_make_module_req(self): write_file(os.path.join(eb.installdir, 'lib', 'libfoo.so'), 'test') shutil.rmtree(os.path.join(eb.installdir, 'lib64')) os.symlink('lib', os.path.join(eb.installdir, 'lib64')) - eb.check_install_lib_symlink() with eb.module_generator.start_module_creation(): guess = eb.make_module_req() if get_module_syntax() == 'Tcl': @@ -616,17 +614,54 @@ def test_make_module_req(self): self.assertEqual(list(eb.module_load_environment), ['PATH', 'LD_LIBRARY_PATH', 'NONPATH']) if get_module_syntax() == 'Tcl': - self.assertTrue(re.match(r"^\nprepend-path\s+PATH\s+\$root/bin\n$", txt, re.M)) - self.assertFalse(re.match(r"^\nprepend-path\s+NONPATH\s+\$root/non_path\n$", txt, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+PATH\s+\$root/bin$", txt, re.M)) + self.assertFalse(re.search(r"^prepend-path\s+NONPATH\s+\$root/non_path$", txt, re.M)) elif get_module_syntax() == 'Lua': - self.assertTrue(re.match(r'^\nprepend_path\("PATH", pathJoin\(root, "bin"\)\)\n$', txt, re.M)) - self.assertFalse(re.match(r'^\nprepend_path\("NONPATH", pathJoin\(root, "non_path"\)\)\n$', txt, re.M)) + self.assertTrue(re.search(r'^prepend_path\("PATH", pathJoin\(root, "bin"\)\)$', txt, re.M)) + self.assertFalse(re.search(r'^prepend_path\("NONPATH", pathJoin\(root, "non_path"\)\)$', txt, re.M)) else: self.fail("Unknown module syntax: %s" % get_module_syntax()) logtxt = read_file(eb.logfile) self.assertTrue(re.search(r"WARNING Non-path variables found in module load env.*NONPATH", logtxt, re.M)) + delattr(eb.module_load_environment, 'NONPATH') + + # make sure that entries that symlink to another directory are retained; + # the test case inspired by the directory structure for old imkl versions (like 2020.4) + remove_dir(eb.installdir) + + # lib/ symlinked to libraries/ + real_libdir = os.path.join(eb.installdir, 'libraries') + mkdir(real_libdir, parents=True) + symlink(real_libdir, os.path.join(eb.installdir, 'lib')) + + # lib/intel64/ symlinked to lib/intel64_lin/ + mkdir(os.path.join(eb.installdir, 'lib', 'intel64_lin'), parents=True) + symlink(os.path.join(eb.installdir, 'lib', 'intel64_lin'), os.path.join(eb.installdir, 'lib', 'intel64')) + + # library file present in lib/intel64 + write_file(os.path.join(eb.installdir, 'lib', 'intel64', 'libfoo.so'), 'libfoo.so') + + # lib64/ symlinked to lib/ + symlink(os.path.join(eb.installdir, 'lib'), os.path.join(eb.installdir, 'lib64')) + + eb.module_load_environment.LD_LIBRARY_PATH = [os.path.join('lib', 'intel64')] + eb.module_load_environment.LIBRARY_PATH = eb.module_load_environment.LD_LIBRARY_PATH + with eb.module_generator.start_module_creation(): + txt = eb.make_module_req() + + if get_module_syntax() == 'Tcl': + self.assertTrue(re.search(r"^prepend-path\s+LD_LIBRARY_PATH\s+\$root/lib/intel64$", txt, re.M)) + self.assertTrue(re.search(r"^prepend-path\s+LIBRARY_PATH\s+\$root/lib/intel64\n$", txt, re.M)) + elif get_module_syntax() == 'Lua': + self.assertTrue(re.search(r'^prepend_path\("LD_LIBRARY_PATH", pathJoin\(root, "lib/intel64"\)\)$', + txt, re.M)) + self.assertTrue(re.search(r'^prepend_path\("LIBRARY_PATH", pathJoin\(root, "lib/intel64"\)\)$', + txt, re.M)) + else: + self.fail("Unknown module syntax: %s" % get_module_syntax()) + # cleanup eb.close_log() os.remove(eb.logfile) @@ -3215,9 +3250,6 @@ def test_expand_module_search_path(self): write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file12.txt'), 'test file 1.2') write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir2', 'file21.txt'), 'test file 2.1') - eb.check_install_lib_symlink() - self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER) - self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH), []) self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_FILES), []) self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_TOP_FILES), []) @@ -3252,13 +3284,8 @@ def test_expand_module_search_path(self): self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_FILES), []) self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_TOP_FILES), []) - # state of install_lib_symlink should not have changed - self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER) - # test just one lib directory os.mkdir(os.path.join(eb.installdir, "lib")) - eb.check_install_lib_symlink() - self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER) self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), []) self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), []) @@ -3269,8 +3296,6 @@ def test_expand_module_search_path(self): # test both lib and lib64 directories os.mkdir(os.path.join(eb.installdir, "lib64")) - eb.check_install_lib_symlink() - self.assertEqual(eb.install_lib_symlink, LibSymlink.NEITHER) self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"]) self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) @@ -3282,17 +3307,15 @@ def test_expand_module_search_path(self): # test lib64 symlinked to lib remove_dir(os.path.join(eb.installdir, "lib64")) os.symlink("lib", os.path.join(eb.installdir, "lib64")) - eb.check_install_lib_symlink() - self.assertEqual(eb.install_lib_symlink, LibSymlink.LIB64_TO_LIB) self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"]) self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), []) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), []) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib64"]) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) + self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["lib", "lib64"]) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["lib", "lib64"]) # test lib symlinked to lib64 remove_dir(os.path.join(eb.installdir, "lib")) @@ -3300,17 +3323,15 @@ def test_expand_module_search_path(self): os.mkdir(os.path.join(eb.installdir, "lib64")) write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib") os.symlink("lib64", os.path.join(eb.installdir, "lib")) - eb.check_install_lib_symlink() - self.assertEqual(eb.install_lib_symlink, LibSymlink.LIB_TO_LIB64) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), []) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), []) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"]) + self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib64"]) self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["lib", "lib64"]) + self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["lib", "lib64"]) def suite():