diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index afe9e58595..ad250ee47f 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -91,11 +91,10 @@ from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files -from easybuild.tools.filetools import dir_contains_files, download_file, encode_class_name, extract_file -from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url -from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs -from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum -from easybuild.tools.filetools import weld_paths, write_file +from easybuild.tools.filetools import download_file, encode_class_name, extract_file, find_backup_name_candidate +from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url, is_binary, is_parent_path +from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir +from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file from easybuild.tools.hooks import ( BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP, @@ -1706,10 +1705,7 @@ def make_module_req(self): mod_req_paths = search_paths self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}") else: - mod_req_paths = [ - expanded_path for unexpanded_path in search_paths - for expanded_path in self.expand_module_search_path(unexpanded_path, path_type=search_paths.type) - ] + mod_req_paths = search_paths.expand_paths(self.installdir) if mod_req_paths: mod_req_paths = nub(mod_req_paths) # remove duplicates @@ -1783,64 +1779,10 @@ def inject_module_extra_paths(self): def expand_module_search_path(self, search_path, path_type=ModEnvVarType.PATH_WITH_FILES): """ - Expand given path glob and return list of suitable paths to be used as search paths: - - Paths must point to existing files/directories - - Relative paths are relative to installation prefix root and are kept relative after expansion - - Absolute paths are kept as absolute paths after expansion - - Follow symlinks and resolve their paths (avoids duplicate paths through symlinks) - - :path_type: ModEnvVarType that controls requirements for population of directories - - PATH: no requirements, can be empty - - PATH_WITH_FILES: must contain at least one file in them (default) - - PATH_WITH_TOP_FILES: increase stricness to require files in top level directory - """ - populated_path_types = ( - ModEnvVarType.PATH_WITH_FILES, - ModEnvVarType.PATH_WITH_TOP_FILES, - ModEnvVarType.STRICT_PATH_WITH_FILES, - ) - - if os.path.isabs(search_path): - abs_glob = search_path - else: - real_installdir = os.path.realpath(self.installdir) - abs_glob = os.path.join(real_installdir, search_path) - - exp_search_paths = glob.glob(abs_glob, recursive=True) - - retained_search_paths = [] - for abs_path in exp_search_paths: - # avoid going through symlink for strict path types - if path_type is ModEnvVarType.STRICT_PATH_WITH_FILES and abs_path != os.path.realpath(abs_path): - self.log.debug( - f"Discarded strict search path '{search_path} of type '{path_type}' that does not correspond " - f"to its real path: {abs_path}" - ) - continue - - if os.path.isdir(abs_path) and path_type in populated_path_types: - # only retain paths to directories that contain at least one file - recursive = path_type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.STRICT_PATH_WITH_FILES) - if not dir_contains_files(abs_path, recursive=recursive): - self.log.debug("Discarded search path to empty directory: %s", abs_path) - continue - - if os.path.isabs(search_path): - retain_path = abs_path - else: - # recover relative path - retain_path = os.path.relpath(os.path.realpath(abs_path), start=real_installdir) - if retain_path == '.': - retain_path = '' # use empty string to represent root of install dir - - if retain_path.startswith('..' + os.path.sep): - raise EasyBuildError( - f"Expansion of search path glob pattern '{search_path}' resulted in a relative path " - f"pointing outside of install directory: {retain_path}" - ) - - retained_search_paths.append(retain_path) - - return retained_search_paths + REMOVED in EasyBuild 5.1, use EasyBlock.module_load_environment.expand_paths instead + """ + msg = "expand_module_search_path is replaced by EasyBlock.module_load_environment.expand_paths" + self.log.nosupport(msg, '5.1') def make_module_req_guess(self): """ diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 1b1b914660..90599af8ba 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -50,7 +50,8 @@ from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars -from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file +from easybuild.tools.filetools import convert_name, dir_contains_files, mkdir, normalize_path, path_matches, read_file +from easybuild.tools.filetools import which, write_file from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -237,6 +238,7 @@ def remove(self, *args): @property def is_path(self): + """Return True for any ModEnvVarType that is a path""" path_like_types = [ ModEnvVarType.PATH, ModEnvVarType.PATH_WITH_FILES, @@ -245,6 +247,81 @@ def is_path(self): ] return self.type in path_like_types + def expand_paths(self, parent): + """ + Expand path glob into list of unique corresponding real paths. + General behaviour: + - Only expand path-like variables + - Paths must point to existing files/directories + - Resolve paths following symlinks into real paths to avoid duplicate + paths through symlinks + - Relative paths are expanded on given parent folder and are kept + relative after expansion + - Absolute paths are kept as absolute paths after expansion + Follow requirements based on current type (ModEnvVarType): + - PATH: no requirements, must exist but can be empty + - PATH_WITH_FILES: must contain at least one file anywhere in subtree + - PATH_WITH_TOP_FILES: must contain files in top level directory of path + - STRICT_PATH_WITH_FILES: given path must expand into its real path and + contain files anywhere in subtree + """ + if not self.is_path: + return None + + populated_path_types = ( + ModEnvVarType.PATH_WITH_FILES, + ModEnvVarType.PATH_WITH_TOP_FILES, + ModEnvVarType.STRICT_PATH_WITH_FILES, + ) + + retained_expanded_paths = [] + real_parent = os.path.realpath(parent) + + for path_glob in self.contents: + abs_glob = path_glob + if not os.path.isabs(path_glob): + abs_glob = os.path.join(real_parent, path_glob) + + expanded_paths = glob.glob(abs_glob, recursive=True) + + for exp_path in expanded_paths: + real_path = os.path.realpath(exp_path) + + if self.type is ModEnvVarType.STRICT_PATH_WITH_FILES and exp_path != real_path: + # avoid going through symlink for strict path types + self.log.debug( + f"Discarded search path '{exp_path} of type '{self.type}' as it does not correspond " + f"to its real path: {real_path}" + ) + continue + + if os.path.isdir(exp_path) and self.type in populated_path_types: + # only retain paths to directories that contain at least one file + recursive = self.type in (ModEnvVarType.PATH_WITH_FILES, ModEnvVarType.STRICT_PATH_WITH_FILES) + if not dir_contains_files(exp_path, recursive=recursive): + self.log.debug(f"Discarded search path '{exp_path}' of type '{self.type}' to empty directory.") + continue + + retain_path = exp_path # no discards, we got a keeper + + if not os.path.isabs(path_glob): + # recover relative path + retain_path = os.path.relpath(real_path, start=real_parent) + # modules use empty string to represent root of install dir + if retain_path == '.': + retain_path = '' + + if retain_path.startswith('..' + os.path.sep): + raise EasyBuildError( + f"Expansion of search path glob pattern '{path_glob}' resulted in a relative path " + f"pointing outside of parent directory: {retain_path}" + ) + + if retain_path not in retained_expanded_paths: + retained_expanded_paths.append(retain_path) + + return retained_expanded_paths + class ModuleLoadEnvironment: """ diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 7262a93bcd..a624704d44 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -54,7 +54,7 @@ from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_dir, remove_file from easybuild.tools.filetools import symlink, verify_checksum, write_file from easybuild.tools.module_generator import module_generator -from easybuild.tools.modules import EnvironmentModules, Lmod, ModEnvVarType, reset_module_caches +from easybuild.tools.modules import EnvironmentModules, Lmod, reset_module_caches from easybuild.tools.version import get_git_revision, this_is_easybuild @@ -682,7 +682,7 @@ def test_make_module_req(self): eb.module_load_environment.PATH = ['bin'] with eb.module_generator.start_module_creation(): - err_regex = "Expansion of search path glob.*pointing outside of install directory.*" + err_regex = "Expansion of search path glob.*pointing outside of parent directory.*" self.assertErrorRegex(EasyBuildError, err_regex, eb.make_module_req) # Test modextrapaths: with absolute + empty paths, appending and custom delimiters @@ -3539,134 +3539,6 @@ def test_create_easyblock_without_logfile(self): os.remove(eb.logfile) - def test_expand_module_search_path(self): - """Testcase for expand_module_search_path""" - top_dir = os.path.abspath(os.path.dirname(__file__)) - toy_ec = os.path.join(top_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') - eb = EasyBlock(EasyConfig(toy_ec)) - eb.installdir = config.install_path() - test_emsp = eb.expand_module_search_path # shortcut - - # create test directories and files - os.makedirs(eb.installdir) - test_directories = ( - 'empty_dir', - 'dir_empty_subdir', - ('dir_empty_subdir', 'empty_subdir'), - 'dir_with_file', - 'dir_full_subdirs', - ('dir_full_subdirs', 'subdir1'), - ('dir_full_subdirs', 'subdir2'), - ) - for path in test_directories: - path_components = (path, ) if isinstance(path, str) else path - os.mkdir(os.path.join(eb.installdir, *path_components)) - - write_file(os.path.join(eb.installdir, 'dir_with_file', 'file.txt'), 'test file') - write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file11.txt'), 'test file 1.1') - write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir1', 'file12.txt'), 'test file 1.2') - write_file(os.path.join(eb.installdir, 'dir_full_subdirs', 'subdir2', 'file21.txt'), 'test file 2.1') - - self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH), []) - self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("nonexistent", ModEnvVarType.PATH_WITH_TOP_FILES), []) - self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH), ["empty_dir"]) - self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("empty_dir", ModEnvVarType.PATH_WITH_TOP_FILES), []) - self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH), ["dir_empty_subdir"]) - self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("dir_empty_subdir", ModEnvVarType.PATH_WITH_TOP_FILES), []) - self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH), ["dir_with_file"]) - self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH_WITH_FILES), ["dir_with_file"]) - self.assertEqual(test_emsp("dir_with_file", ModEnvVarType.PATH_WITH_TOP_FILES), ["dir_with_file"]) - self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH), ["dir_full_subdirs"]) - self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH_WITH_FILES), ["dir_full_subdirs"]) - self.assertEqual(test_emsp("dir_full_subdirs", ModEnvVarType.PATH_WITH_TOP_FILES), []) - - # test globs - ref_expanded_paths = ["dir_empty_subdir/empty_subdir"] - self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH), ref_expanded_paths) - self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("dir_empty_subdir/*", ModEnvVarType.PATH_WITH_TOP_FILES), []) - ref_expanded_paths = ["dir_full_subdirs/subdir1", "dir_full_subdirs/subdir2"] - self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH)), ref_expanded_paths) - self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH_WITH_FILES)), ref_expanded_paths) - self.assertEqual(sorted(test_emsp("dir_full_subdirs/*", ModEnvVarType.PATH_WITH_TOP_FILES)), ref_expanded_paths) - ref_expanded_paths = ["dir_full_subdirs/subdir2/file21.txt"] - self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH), ref_expanded_paths) - self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH_WITH_FILES), ref_expanded_paths) - self.assertEqual(test_emsp("dir_full_subdirs/subdir2/*", ModEnvVarType.PATH_WITH_TOP_FILES), ref_expanded_paths) - self.assertEqual(test_emsp("nonexistent/*", True), []) - self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH), []) - self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("nonexistent/*", ModEnvVarType.PATH_WITH_TOP_FILES), []) - - # test just one lib directory - os.mkdir(os.path.join(eb.installdir, "lib")) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), []) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), []) - write_file(os.path.join(eb.installdir, "lib", "libtest.so"), "not actually a lib") - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - - # test both lib and lib64 directories - os.mkdir(os.path.join(eb.installdir, "lib64")) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib") - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["lib", "lib64"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["lib", "lib64"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["lib", "lib64"]) - - # test lib64 symlinked to lib - remove_dir(os.path.join(eb.installdir, "lib64")) - os.symlink("lib", os.path.join(eb.installdir, "lib64")) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib", "lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib", "lib"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib", "lib"]) - - # test lib symlinked to lib64 - remove_dir(os.path.join(eb.installdir, "lib")) - remove_file(os.path.join(eb.installdir, "lib64")) - os.mkdir(os.path.join(eb.installdir, "lib64")) - write_file(os.path.join(eb.installdir, "lib64", "libtest.so"), "not actually a lib") - os.symlink("lib64", os.path.join(eb.installdir, "lib")) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["lib64"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["lib64"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH), ["lib64", "lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES), ["lib64", "lib64"]) - self.assertEqual(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES), ["lib64", "lib64"]) - - # test both lib and lib64 symlinked to some other folder - remove_dir(os.path.join(eb.installdir, "lib64")) - remove_file(os.path.join(eb.installdir, "lib")) - os.mkdir(os.path.join(eb.installdir, "some_dir")) - write_file(os.path.join(eb.installdir, "some_dir", "libtest.so"), "not actually a lib") - os.symlink("some_dir", os.path.join(eb.installdir, "lib")) - os.symlink("some_dir", os.path.join(eb.installdir, "lib64")) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH), ["some_dir"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_FILES), ["some_dir"]) - self.assertEqual(test_emsp("lib", ModEnvVarType.PATH_WITH_TOP_FILES), ["some_dir"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH), ["some_dir"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_FILES), ["some_dir"]) - self.assertEqual(test_emsp("lib64", ModEnvVarType.PATH_WITH_TOP_FILES), ["some_dir"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH)), ["some_dir", "some_dir"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_FILES)), ["some_dir", "some_dir"]) - self.assertEqual(sorted(test_emsp("lib*", ModEnvVarType.PATH_WITH_TOP_FILES)), ["some_dir", "some_dir"]) - def suite(): """ return all the tests in this file """ diff --git a/test/framework/modules.py b/test/framework/modules.py index 6080366fa6..dc111930ac 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1682,6 +1682,219 @@ def test_module_environment_variable(self): self.assertEqual(mod_envar.contents, ['new_path_1', 'new_path_2']) self.assertRaises(TypeError, mod_envar.update, 'arg1', 'arg2') + def test_module_environment_variable_expand_paths(self): + """Testcase for ModuleEnvironmentVariable.expand_paths""" + + # create test directories and files + installdir = tempfile.mkdtemp() + test_directories = ( + 'empty_dir', + 'dir_empty_subdir', + ('dir_empty_subdir', 'empty_subdir'), + 'dir_with_file', + 'dir_full_subdirs', + ('dir_full_subdirs', 'subdir1'), + ('dir_full_subdirs', 'subdir2'), + ) + for path in test_directories: + path_components = (path, ) if isinstance(path, str) else path + os.mkdir(os.path.join(installdir, *path_components)) + + write_file(os.path.join(installdir, 'dir_with_file', 'file.txt'), 'test file') + write_file(os.path.join(installdir, 'dir_full_subdirs', 'subdir1', 'file11.txt'), 'test file 1.1') + write_file(os.path.join(installdir, 'dir_full_subdirs', 'subdir1', 'file12.txt'), 'test file 1.2') + write_file(os.path.join(installdir, 'dir_full_subdirs', 'subdir2', 'file21.txt'), 'test file 2.1') + + def assert_expanded_paths(test_paths, references, installdir): + mod_envar = mod.ModuleEnvironmentVariable(test_paths) + for var_type, ref_value in references: + mod_envar.type = var_type + self.assertEqual(sorted(mod_envar.expand_paths(installdir)), sorted(ref_value)) + + # test simple paths + test_paths = ['nonexistent', 'empty_dir', 'dir_empty_subdir', 'dir_with_file', 'dir_full_subdirs'] + references = ( + (mod.ModEnvVarType.PATH, ['empty_dir', 'dir_empty_subdir', 'dir_with_file', 'dir_full_subdirs']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['dir_with_file', 'dir_full_subdirs']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['dir_with_file']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['dir_with_file', 'dir_full_subdirs']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test globs + test_paths = ['dir_empty_subdir/*', 'dir_full_subdirs/*', 'dir_full_subdirs/subdir2/*', 'nonexistent/*'] + references = ( + (mod.ModEnvVarType.PATH, [ + "dir_empty_subdir/empty_subdir", + "dir_full_subdirs/subdir1", + "dir_full_subdirs/subdir2", + "dir_full_subdirs/subdir2/file21.txt", + ]), + (mod.ModEnvVarType.PATH_WITH_FILES, [ + "dir_full_subdirs/subdir1", + "dir_full_subdirs/subdir2", + "dir_full_subdirs/subdir2/file21.txt", + ]), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, [ + "dir_full_subdirs/subdir1", + "dir_full_subdirs/subdir2", + "dir_full_subdirs/subdir2/file21.txt", + ]), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, [ + "dir_full_subdirs/subdir1", + "dir_full_subdirs/subdir2", + "dir_full_subdirs/subdir2/file21.txt", + ]), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test just one lib directory + os.mkdir(os.path.join(installdir, "lib")) + + test_paths = ['lib', 'lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, []), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, []), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + + write_file(os.path.join(installdir, 'lib', 'libtest.so'), "not actually a lib") + test_paths = ['lib', 'lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test both lib and lib64 directories + os.mkdir(os.path.join(installdir, "lib64")) + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['lib', 'lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib']), + ) + assert_expanded_paths(test_paths, references, installdir) + + write_file(os.path.join(installdir, "lib64", "libtest.so"), "not actually a lib") + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['lib', 'lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib', 'lib64']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib', 'lib64']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib', 'lib64']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test lib64 symlinked to lib + remove_dir(os.path.join(installdir, "lib64")) + os.symlink("lib", os.path.join(installdir, "lib64")) + test_paths = ['lib'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib']), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['lib']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test lib symlinked to lib64 + remove_dir(os.path.join(installdir, "lib")) + remove_file(os.path.join(installdir, "lib64")) + os.mkdir(os.path.join(installdir, "lib64")) + write_file(os.path.join(installdir, "lib64", "libtest.so"), "not actually a lib") + os.symlink("lib64", os.path.join(installdir, "lib")) + test_paths = ['lib'] + references = ( + (mod.ModEnvVarType.PATH, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib64']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib64']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib64']), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['lib64']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['lib64']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, ['lib64']), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test both lib and lib64 symlinked to some other folder + remove_dir(os.path.join(installdir, "lib64")) + remove_file(os.path.join(installdir, "lib")) + os.mkdir(os.path.join(installdir, "some_dir")) + write_file(os.path.join(installdir, "some_dir", "libtest.so"), "not actually a lib") + os.symlink("some_dir", os.path.join(installdir, "lib")) + os.symlink("some_dir", os.path.join(installdir, "lib64")) + test_paths = ['lib'] + references = ( + (mod.ModEnvVarType.PATH, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['some_dir']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib64'] + references = ( + (mod.ModEnvVarType.PATH, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['some_dir']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + test_paths = ['lib*'] + references = ( + (mod.ModEnvVarType.PATH, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_FILES, ['some_dir']), + (mod.ModEnvVarType.PATH_WITH_TOP_FILES, ['some_dir']), + (mod.ModEnvVarType.STRICT_PATH_WITH_FILES, []), + ) + assert_expanded_paths(test_paths, references, installdir) + + # test folder symlinked to folder outside installdir + os.symlink("/bin", os.path.join(installdir, "external_bin")) + test_paths = ['external_bin'] + mod_envar = mod.ModuleEnvironmentVariable(test_paths) + mod_envar.type = mod.ModEnvVarType.PATH + self.assertRaises(EasyBuildError, mod_envar.expand_paths, installdir) + mod_envar.type = mod.ModEnvVarType.PATH_WITH_FILES + self.assertRaises(EasyBuildError, mod_envar.expand_paths, installdir) + mod_envar.type = mod.ModEnvVarType.PATH_WITH_TOP_FILES + self.assertRaises(EasyBuildError, mod_envar.expand_paths, installdir) + mod_envar.type = mod.ModEnvVarType.STRICT_PATH_WITH_FILES + self.assertEqual(mod_envar.expand_paths(installdir), []) + def test_module_load_environment(self): """Test for ModuleLoadEnvironment object""" mod_load_env = mod.ModuleLoadEnvironment()