diff --git a/easybuild/base/testing.py b/easybuild/base/testing.py index 55275c5875..509f59e63e 100644 --- a/easybuild/base/testing.py +++ b/easybuild/base/testing.py @@ -117,6 +117,11 @@ def assertNotExists(self, path, msg=None): msg = "'%s' should not exist" % path self.assertFalse(os.path.exists(path), msg) + def assertAllExist(self, paths, msg=None): + """Assert that all paths in the given list exist""" + for path in paths: + self.assertExists(path, msg) + def setUp(self): """Prepare test case.""" super(TestCase, self).setUp() diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index cfa25052bd..c6f0f8d6bb 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -48,6 +48,7 @@ import inspect import json import os +import random import re import stat import sys @@ -57,6 +58,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import datetime from enum import Enum +from string import ascii_letters from textwrap import indent import easybuild.tools.environment as env @@ -79,17 +81,19 @@ from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES # noqa from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import MOD_SEARCH_PATH_HEADERS, PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS -from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath +from easybuild.tools.config import build_option, build_path, get_failed_install_build_dirs_path +from easybuild.tools.config import get_failed_install_logs_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock -from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info -from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file -from easybuild.tools.filetools import encode_class_name, extract_file, find_backup_name_candidate -from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url -from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir -from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink +from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock +from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files +from easybuild.tools.filetools import dir_contains_files, download_file, encode_class_name, extract_file +from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url +from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs +from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum +from easybuild.tools.filetools import weld_paths, write_file from easybuild.tools.hooks import ( BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP, @@ -4438,6 +4442,54 @@ def print_dry_run_note(loc, silent=True): dry_run_msg(msg, silent=silent) +def copy_build_dirs_logs_failed_install(application_log, silent, app, easyconfig): + """ + Copy build directories and log files for failed installation (if desired) + """ + logs_path = get_failed_install_logs_path(easyconfig) + build_dirs_path = get_failed_install_build_dirs_path(easyconfig) + + # there may be multiple log files, or the file name may be different due to zipping + logs = glob.glob(f"{application_log}*") + + timestamp = time.strftime('%Y%m%d-%H%M%S') + salt = ''.join(random.choices(ascii_letters, k=5)) + unique_subdir = f'{timestamp}-{salt}' + + operation_args = [] + + if logs_path and logs: + logs_path = os.path.join(logs_path, unique_subdir) + + if is_parent_path(app.builddir, logs_path): + msg = "Path to copy log files of failed installs to is subdirectory of build directory; not copying" + print_warning(msg, log=_log, silent=silent) + else: + msg = f"Log file(s) of failed installation copied to {logs_path}" + operation_args.append((copy_file, logs, logs_path, msg)) + + if build_dirs_path and os.path.isdir(app.builddir): + build_dirs_path = os.path.join(build_dirs_path, unique_subdir) + + if is_parent_path(app.builddir, build_dirs_path): + msg = "Path to copy build dirs of failed installs to is subdirectory of build directory; not copying" + print_warning(msg, log=_log, silent=silent) + else: + msg = f"Build directory of failed installation copied to {build_dirs_path}" + + def operation(src, dest): + copy_dir(src, dest, dirs_exist_ok=True) + + operation_args.append((operation, [app.builddir], build_dirs_path, msg)) + + persistence_paths = create_non_existing_paths(target_path for (_, _, target_path, _) in operation_args) + + for idx, (operation, paths, _, msg) in enumerate(operation_args): + for path in paths: + operation(path, persistence_paths[idx]) + print_msg(msg, log=_log, silent=silent) + + def build_and_install_one(ecdict, init_env): """ Build the software @@ -4696,6 +4748,9 @@ def ensure_writable_log_dir(log_dir): logs = glob.glob('%s*' % application_log) print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent) + if not success: + copy_build_dirs_logs_failed_install(application_log, silent, app, ecdict['ec']) + del app return (success, application_log, error_msg, exit_code) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 9f753d3819..4a853a75c9 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -505,6 +505,8 @@ class ConfigurationVariables(BaseConfigurationVariables): 'buildpath', 'config', 'containerpath', + 'failed_install_build_dirs_path', + 'failed_install_logs_path', 'installpath', 'installpath_modules', 'installpath_software', @@ -877,6 +879,42 @@ def log_path(ec=None): return log_file_format(return_directory=True, ec=ec, date=date, timestamp=timestamp) +def get_failed_install_build_dirs_path(ec): + """ + Return the location where the build directory is copied to if installation failed + + :param ec: dict-like value with 'name' and 'version' keys defined + """ + base_path = ConfigurationVariables()['failed_install_build_dirs_path'] + if not base_path: + return None + + try: + name, version = ec['name'], ec['version'] + except KeyError: + raise EasyBuildError("The 'name' and 'version' keys are required.") + + return os.path.join(base_path, f'{name}-{version}') + + +def get_failed_install_logs_path(ec): + """ + Return the location where log files are copied to if installation failed + + :param ec: dict-like value with 'name' and 'version' keys defined + """ + base_path = ConfigurationVariables()['failed_install_logs_path'] + if not base_path: + return None + + try: + name, version = ec['name'], ec['version'] + except KeyError: + raise EasyBuildError("The 'name' and 'version' keys are required.") + + return os.path.join(base_path, f'{name}-{version}') + + def get_build_log_path(): """ Return (temporary) directory for build log diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 033c88a65d..88859f8b59 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -603,6 +603,19 @@ def normalize_path(path): return start_slashes + os.path.sep.join(filtered_comps) +def is_parent_path(path1, path2): + """ + Return True if path1 is a prefix of path2 + + :param path1: absolute or relative path + :param path2: absolute or relative path + """ + path1 = os.path.realpath(path1) + path2 = os.path.realpath(path2) + common_path = os.path.commonprefix([path1, path2]) + return common_path == path1 + + def is_alt_pypi_url(url): """Determine whether specified URL is already an alternative PyPI URL, i.e. whether it contains a hash.""" # example: .../packages/5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf/easybuild-2.7.0.tar.gz @@ -3096,3 +3109,69 @@ def create_unused_dir(parent_folder, name): set_gid_sticky_bits(path, recursive=True) return path + + +def get_first_non_existing_parent_path(path): + """ + Get first directory that does not exist, starting at path and going up. + """ + path = os.path.abspath(path) + + non_existing_parent = None + while not os.path.exists(path): + non_existing_parent = path + path = os.path.dirname(path) + + return non_existing_parent + + +def create_non_existing_paths(paths, max_tries=10000): + """ + Create directories with given paths (including the parent directories). + When a directory in the same location for any of the specified paths already exists, + then the suffix '_' is appended , with i iteratively picked between 0 and (max_tries-1), + until an index is found so that all required paths are non-existing. + All created directories have the same suffix. + + :param paths: list of directory paths to be created + :param max_tries: maximum number of tries before failing + """ + paths = [os.path.abspath(p) for p in paths] + for idx_path, path in enumerate(paths): + for idx_parent, parent in enumerate(paths): + if idx_parent != idx_path and is_parent_path(parent, path): + raise EasyBuildError(f"Path '{parent}' is a parent path of '{path}'.") + + first_non_existing_parent_paths = [get_first_non_existing_parent_path(p) for p in paths] + + non_existing_paths = paths + all_paths_created = False + suffix = -1 + while suffix < max_tries and not all_paths_created: + tried_paths = [] + if suffix >= 0: + non_existing_paths = [f'{p}_{suffix}' for p in paths] + try: + for path in non_existing_paths: + tried_paths.append(path) + # os.makedirs will raise OSError if directory already exists + os.makedirs(path) + all_paths_created = True + except OSError as err: + # Distinguish between error due to existing folder and anything else + if not os.path.exists(tried_paths[-1]): + raise EasyBuildError("Failed to create directory %s: %s", tried_paths[-1], err) + remove(tried_paths[:-1]) + except BaseException as err: + remove(tried_paths) + raise err + suffix += 1 + + if not all_paths_created: + raise EasyBuildError(f"Exceeded maximum number of attempts ({max_tries}) to generate non-existing paths") + + # set group ID and sticky bits, if desired + for path in first_non_existing_parent_paths: + set_gid_sticky_bits(path, recursive=True) + + return non_existing_paths diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 5e9559f6cc..dca73c0329 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -87,7 +87,7 @@ from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.environment import restore_env, unset_env_vars from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, get_cwd -from easybuild.tools.filetools import install_fake_vsc, move_file, which +from easybuild.tools.filetools import install_fake_vsc, move_file, which, is_parent_path from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS @@ -592,6 +592,12 @@ def config_options(self): [DEFAULT_ENVVAR_USERS_MODULES]), 'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata " "for external modules (INI format)", 'strlist', 'store', None), + 'failed-install-build-dirs-path': ("Location where build directories are copied if installation fails; " + "an empty value disables copying of build directories", + None, 'store', None, {'metavar': "PATH"}), + 'failed-install-logs-path': ("Location where log files are copied if installation fails; " + "an empty value disables copying of log files", + None, 'store', None, {'metavar': "PATH"}), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), 'ignore-dirs': ("Directory names to ignore when searching for files/dirs", 'strlist', 'store', ['.git', '.svn']), @@ -1217,22 +1223,23 @@ def _postprocess_config(self): # to avoid incorrect paths being used when EasyBuild changes the current working directory # (see https://github.com/easybuilders/easybuild-framework/issues/3619); # ensuring absolute paths for 'robot' is handled separately below, - # because we need to be careful with the argument pass to --robot; + # because we need to be careful with the argument passed to --robot; # note: repositorypath is purposely not listed here, because it's a special case: # - the value could consist of a 2-tuple (, ); # - the could also specify the location of a *remote* (Git( repository, # which can be done in variety of formats (git@:/), https://, etc.) # (see also https://github.com/easybuilders/easybuild-framework/issues/3892); - path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath', - 'installpath_modules', 'installpath_software', 'prefix', 'packagepath', - 'robot_paths', 'sourcepath'] + path_opt_names = ['buildpath', 'containerpath', 'failed_install_build_dirs_path', 'failed_install_logs_path', + 'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software', + 'prefix', 'packagepath', 'robot_paths', 'sourcepath'] for opt_name in path_opt_names: self._ensure_abs_path(opt_name) if self.options.prefix is not None: - # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account - # in the legacy-style configuration, repository is initialised in configuration file itself + # prefix applies to selected path configuration options; + # repository has to be reinitialised to take new repositorypath in account; + # in the legacy-style configuration, repository is initialised in configuration file itself; path_opts = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'repository', 'repositorypath', 'sourcepath'] for dest in path_opts: @@ -1293,6 +1300,20 @@ def _postprocess_config(self): if self.options.inject_checksums or self.options.inject_checksums_to_json: self.options.pre_create_installdir = False + # Prevent that build directories and logs for failed installations are copied to location for build directories + if self.options.buildpath and self.options.failed_install_logs_path: + if is_parent_path(self.options.buildpath, self.options.failed_install_logs_path): + raise EasyBuildError( + f"The --failed-install-logs-path ('{self.options.failed_install_logs_path}') " + f"cannot reside in a subdirectory of the --buildpath ('{self.options.buildpath}')" + ) + if self.options.buildpath and self.options.failed_install_build_dirs_path: + if is_parent_path(self.options.buildpath, self.options.failed_install_build_dirs_path): + raise EasyBuildError( + f"The --failed-install-build-dirs-path ('{self.options.failed_install_build_dirs_path}') " + f"cannot reside in a subdirectory of the --buildpath ('{self.options.buildpath}')" + ) + def _postprocess_list_avail(self): """Create all the additional info that can be requested (exit at the end)""" msg = '' diff --git a/test/framework/filetools.py b/test/framework/filetools.py index b72f3ded25..492d8fce85 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -476,6 +476,43 @@ def test_normalize_path(self): self.assertEqual(ft.normalize_path('/././foo//bar/././baz/'), '/foo/bar/baz') self.assertEqual(ft.normalize_path('//././foo//bar/././baz/'), '//foo/bar/baz') + def test_is_parent_path(self): + """Test is_parent_path""" + self.assertTrue(ft.is_parent_path('/foo/bar', '/foo/bar/test0')) + self.assertTrue(ft.is_parent_path('/foo/bar', '/foo/bar/test0/test1')) + self.assertTrue(ft.is_parent_path('/foo/bar', '/foo/bar')) + self.assertFalse(ft.is_parent_path('/foo/bar/test0', '/foo/bar')) + self.assertFalse(ft.is_parent_path('/foo/bar', '/foo/test')) + + # Check that trailing slashes are ignored + self.assertTrue(ft.is_parent_path('/foo/bar/', '/foo/bar')) + self.assertTrue(ft.is_parent_path('/foo/bar', '/foo/bar/')) + self.assertTrue(ft.is_parent_path('/foo/bar/', '/foo/bar/')) + + # Check that is also accepts relative paths + self.assertTrue(ft.is_parent_path('foo/bar', 'foo/bar/test0')) + self.assertTrue(ft.is_parent_path('foo/bar', 'foo/bar/test0/test1')) + self.assertTrue(ft.is_parent_path('foo/bar', 'foo/bar')) + self.assertFalse(ft.is_parent_path('foo/bar/test0', 'foo/bar')) + self.assertFalse(ft.is_parent_path('foo/bar', 'foo/test')) + + # Check that relative paths are accounted + self.assertTrue(ft.is_parent_path('foo/../baz', 'bar/../baz')) + + # Check that symbolic links are accounted + ft.mkdir(os.path.join(self.test_prefix, 'base')) + ft.mkdir(os.path.join(self.test_prefix, 'base', 'concrete')) + ft.symlink( + os.path.join(self.test_prefix, 'base', 'concrete'), + os.path.join(self.test_prefix, 'base', 'link') + ) + self.assertTrue( + ft.is_parent_path( + os.path.join(self.test_prefix, 'base', 'link'), + os.path.join(self.test_prefix, 'base', 'concrete', 'file') + ) + ) + def test_det_file_size(self): """Test det_file_size function.""" @@ -3707,6 +3744,30 @@ def test_set_gid_sticky_bits(self): self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + def test_get_first_non_existing_parent_path(self): + """Test get_first_non_existing_parent_path function.""" + root_path = os.path.join(self.test_prefix, 'a') + target_path = os.path.join(self.test_prefix, 'a', 'b', 'c') + ft.mkdir(root_path, parents=True) + first_non_existing_parent = ft.get_first_non_existing_parent_path(target_path) + self.assertEqual(first_non_existing_parent, os.path.join(self.test_prefix, 'a', 'b')) + ft.remove_dir(root_path) + + # Use a reflexive parent relation + root_path = os.path.join(self.test_prefix, 'a', 'b') + target_path = os.path.join(self.test_prefix, 'a', 'b', 'c') + ft.mkdir(root_path, parents=True) + first_non_existing_parent = ft.get_first_non_existing_parent_path(target_path) + self.assertEqual(first_non_existing_parent, os.path.join(self.test_prefix, 'a', 'b', 'c')) + ft.remove_dir(root_path) + + root_path = os.path.join(self.test_prefix, 'a', 'b', 'c') + target_path = os.path.join(self.test_prefix, 'a', 'b', 'c') + ft.mkdir(root_path, parents=True) + first_non_existing_parent = ft.get_first_non_existing_parent_path(target_path) + self.assertEqual(first_non_existing_parent, None) + ft.remove_dir(root_path) + def test_create_unused_dir(self): """Test create_unused_dir function.""" path = ft.create_unused_dir(self.test_prefix, 'folder') @@ -3745,6 +3806,234 @@ def test_create_unused_dir(self): self.assertEqual(path, os.path.join(self.test_prefix, 'file_0')) self.assertExists(path) + def test_create_non_existing_paths(self): + """Test create_non_existing_paths function.""" + test_root = os.path.join(self.test_prefix, 'test_create_non_existing_paths') + + ft.mkdir(test_root) + requested_paths = [ + os.path.join(test_root, 'folder_a'), + os.path.join(test_root, 'folder_b'), + ] + paths = ft.create_non_existing_paths(requested_paths) + self.assertEqual(paths, requested_paths) + self.assertAllExist(paths) + ft.remove_dir(test_root) + + # Repeat with existing folder(s) should create new ones + ft.mkdir(test_root) + requested_paths = [ + os.path.join(test_root, 'folder_a'), + os.path.join(test_root, 'folder_b'), + ] + for path in requested_paths: + ft.mkdir(path) + for i in range(10): + paths = ft.create_non_existing_paths(requested_paths) + self.assertEqual(paths, [f'{p}_{i}' for p in requested_paths]) + self.assertAllExist(paths) + ft.remove_dir(test_root) + + # Add a suffix in both directories if a suffix already exists + ft.mkdir(test_root) + requested_paths = [ + os.path.join(test_root, 'existing_a'), + os.path.join(test_root, 'existing_b'), + ] + ft.mkdir(os.path.join(test_root, 'existing_b')) + paths = ft.create_non_existing_paths(requested_paths) + self.assertEqual(paths, [f'{p}_0' for p in requested_paths]) + self.assertNotExists(os.path.join(test_root, 'existing_a')) + self.assertAllExist(paths) + ft.remove_dir(test_root) + + # Skip suffix if a directory with the suffix already exists + ft.mkdir(test_root) + existing_suffix = 1 + requested_paths = [ + os.path.join(test_root, 'existing_suffix_a'), + os.path.join(test_root, 'existing_suffix_b'), + ] + + ft.mkdir(os.path.join(test_root, f'existing_suffix_b_{existing_suffix}')) + + def expected_suffix(n_calls_to_create_non_existing_paths): + if n_calls_to_create_non_existing_paths == 0: + return "" + new_suffix = n_calls_to_create_non_existing_paths - 1 + if n_calls_to_create_non_existing_paths > existing_suffix: + new_suffix += 1 + return f"_{new_suffix}" + + for i in range(3): + paths = ft.create_non_existing_paths(requested_paths) + self.assertEqual(paths, [p + expected_suffix(i) for p in requested_paths]) + self.assertAllExist(paths) + self.assertNotExists(os.path.join(test_root, f'existing_suffix_a_{existing_suffix}')) + self.assertExists(os.path.join(test_root, f'existing_suffix_b_{existing_suffix}')) + + ft.remove_dir(test_root) + + # Support creation of parent directories + ft.mkdir(test_root) + requested_paths = [os.path.join(test_root, 'parent_folder', 'folder')] + paths = ft.create_non_existing_paths(requested_paths) + self.assertEqual(paths, requested_paths) + self.assertAllExist(paths) + ft.remove_dir(test_root) + + # Not influenced by similar folder + ft.mkdir(test_root) + requested_paths = [os.path.join(test_root, 'folder_a2')] + paths = ft.create_non_existing_paths(requested_paths) + self.assertEqual(paths, requested_paths) + self.assertAllExist(paths) + for i in range(10): + paths = ft.create_non_existing_paths(requested_paths) + self.assertEqual(paths, [f'{p}_{i}' for p in requested_paths]) + self.assertAllExist(paths) + ft.remove_dir(test_root) + + # Fail cleanly if passed a readonly folder + ft.mkdir(test_root) + readonly_dir = os.path.join(test_root, 'ro_folder') + ft.mkdir(readonly_dir) + old_perms = os.lstat(readonly_dir)[stat.ST_MODE] + ft.adjust_permissions(readonly_dir, stat.S_IREAD | stat.S_IEXEC, relative=False) + requested_path = [os.path.join(readonly_dir, 'new_folder')] + try: + self.assertErrorRegex( + EasyBuildError, "Failed to create directory", + ft.create_non_existing_paths, requested_path + ) + finally: + ft.adjust_permissions(readonly_dir, old_perms, relative=False) + ft.remove_dir(test_root) + + # Fail if the number of attempts to create the directory is exceeded + ft.mkdir(test_root) + requested_paths = [os.path.join(test_root, 'attempt')] + ft.mkdir(os.path.join(test_root, 'attempt')) + ft.mkdir(os.path.join(test_root, 'attempt_0')) + ft.mkdir(os.path.join(test_root, 'attempt_1')) + ft.mkdir(os.path.join(test_root, 'attempt_2')) + ft.mkdir(os.path.join(test_root, 'attempt_3')) + max_tries = 4 + self.assertErrorRegex( + EasyBuildError, + rf"Exceeded maximum number of attempts \({max_tries}\) to generate non-existing paths", + ft.create_non_existing_paths, + requested_paths, max_tries=max_tries + ) + ft.remove_dir(test_root) + + # Ignore files same as folders. So first just create a file with no contents + ft.mkdir(test_root) + requested_path = os.path.join(test_root, 'file') + ft.write_file(requested_path, '') + paths = ft.create_non_existing_paths([requested_path]) + self.assertEqual(paths, [requested_path + '_0']) + self.assertAllExist(paths) + ft.remove_dir(test_root) + + # Deny creation of nested directories + requested_paths = [ + os.path.join(test_root, 'foo/bar'), + os.path.join(test_root, 'foo/bar/baz'), + ] + self.assertErrorRegex( + EasyBuildError, + "Path '.*/foo/bar' is a parent path of '.*/foo/bar/baz'", + ft.create_non_existing_paths, + requested_paths + ) + self.assertNotExists(test_root) # Fail early, do not create intermediate directories + + requested_paths = [ + os.path.join(test_root, 'foo/bar/baz'), + os.path.join(test_root, 'foo/bar'), + ] + self.assertErrorRegex( + EasyBuildError, + "Path '.*/foo/bar' is a parent path of '.*/foo/bar/baz'", + ft.create_non_existing_paths, + requested_paths + ) + self.assertNotExists(test_root) # Fail early, do not create intermediate directories + + requested_paths = [ + os.path.join(test_root, 'foo/bar'), + os.path.join(test_root, 'foo/bar'), + ] + self.assertErrorRegex( + EasyBuildError, + "Path '.*/foo/bar' is a parent path of '.*/foo/bar'", + ft.create_non_existing_paths, + requested_paths + ) + self.assertNotExists(test_root) # Fail early, do not create intermediate directories + + # Allow creation of non-nested directories + ft.mkdir(test_root) + requested_paths = [ + os.path.join(test_root, 'nested/foo/bar'), + os.path.join(test_root, 'nested/foo/baz'), + os.path.join(test_root, 'nested/buz'), + ] + paths = ft.create_non_existing_paths(requested_paths) + self.assertEqual(paths, requested_paths) + self.assertAllExist(paths) + ft.remove_dir(test_root) + + # Test that permissions are set in single directories + ft.mkdir(test_root, set_gid=False, sticky=False) + init_config(build_options={'set_gid_bit': True, 'sticky_bit': True}) + requested_path = os.path.join(test_root, 'directory') + paths = ft.create_non_existing_paths([requested_path]) + self.assertEqual(len(paths), 1) + dir_perms = os.lstat(paths[0])[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID) + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX) + init_config(build_options={'set_gid_bit': None, 'sticky_bit': None}) + ft.remove_dir(test_root) + + # Test that permissions are set correctly across a whole path + ft.mkdir(test_root, set_gid=False, sticky=False) + init_config(build_options={'set_gid_bit': True, 'sticky_bit': True}) + requested_path = os.path.join(test_root, 'directory', 'subdirectory') + paths = ft.create_non_existing_paths([requested_path]) + self.assertEqual(len(paths), 1) + tested_paths = [ + os.path.join(test_root, 'directory'), + os.path.join(test_root, 'directory', 'subdirectory'), + ] + for path in tested_paths: + dir_perms = os.lstat(path)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID, f"set_gid bit should be set for {path}") + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX, f"sticky bit should be set for {path}") + init_config(build_options={'set_gid_bit': None, 'sticky_bit': None}) + ft.remove_dir(test_root) + + # Test that existing directory permissions are not modified + ft.mkdir(test_root) + init_config(build_options={'set_gid_bit': True, 'sticky_bit': True}) + existing_parent = os.path.join(test_root, 'directory') + requested_path = os.path.join(existing_parent, 'subdirectory') + + ft.mkdir(existing_parent, set_gid=False, sticky=False) + paths = ft.create_non_existing_paths([requested_path]) + self.assertEqual(len(paths), 1) + + dir_perms = os.lstat(paths[0])[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, stat.S_ISGID, f"set_gid bit should be set for {path}") + self.assertEqual(dir_perms & stat.S_ISVTX, stat.S_ISVTX, f"sticky bit should be set for {path}") + + dir_perms = os.lstat(existing_parent)[stat.ST_MODE] + self.assertEqual(dir_perms & stat.S_ISGID, 0, f"set_gid bit should not be set for {path}") + self.assertEqual(dir_perms & stat.S_ISVTX, 0, f"sticky bit should not be set for {path}") + init_config(build_options={'set_gid_bit': None, 'sticky_bit': None}) + ft.remove_dir(test_root) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/options.py b/test/framework/options.py index dfcf6c9813..43e97a02f8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1657,6 +1657,52 @@ def test_dry_run(self): regex = re.compile(r" \* \[%s\] \S+%s \(module: %s\)" % (mark, ec, mod), re.M) self.assertTrue(regex.search(logtxt), "Found match for pattern %s in '%s'" % (regex.pattern, logtxt)) + def test_persistence_copying_restrictions(self): + """ + Test that EasyBuild fails when instructed to move logs or artifacts inside the build directory + + Moving log files or artifacts inside the build directory modifies the build artifacts, and in the case of + build artifacts it is also copying directories into themselves. + """ + base_args = [ + 'gzip-1.4-GCC-4.6.3.eb', + '--dry-run', + '--robot', + ] + + def test_eb_with(option_flag, is_valid): + with tempfile.TemporaryDirectory() as root_dir: + build_dir = os.path.join(root_dir, 'build_dir') + if is_valid: + persist_path = os.path.join(root_dir, 'persist_dir') + else: + persist_path = os.path.join(root_dir, 'build_dir', 'persist_dir') + + extra_args = [ + f"--buildpath={build_dir}", + f"{option_flag}={persist_path}", + ] + + pattern = rf"The {option_flag} \(.*\) cannot reside in a subdirectory of the --buildpath \(.*\)" + + args = base_args + args.extend(extra_args) + + if is_valid: + try: + self.eb_main(args, raise_error=True) + except EasyBuildError: + self.fail( + "Should not fail with --buildpath='{build_dir}' and {option_flag}='{persist_path}'." + ) + else: + self.assertErrorRegex(EasyBuildError, pattern, self.eb_main, args, raise_error=True) + + test_eb_with(option_flag='--failed-install-logs-path', is_valid=True) + test_eb_with(option_flag='--failed-install-logs-path', is_valid=False) + test_eb_with(option_flag='--failed-install-build-dirs-path', is_valid=True) + test_eb_with(option_flag='--failed-install-build-dirs-path', is_valid=False) + def test_missing(self): """Test use of --missing/-M.""" @@ -5440,7 +5486,14 @@ def test_prefix_option(self): regex = re.compile(r"(?P\S*).*%s.*" % self.test_prefix, re.M) - expected = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'prefix', 'repositorypath'] + expected = [ + 'buildpath', + 'containerpath', + 'installpath', + 'packagepath', + 'prefix', + 'repositorypath', + ] self.assertEqual(sorted(regex.findall(txt)), expected) def test_dump_env_script(self): diff --git a/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch new file mode 100644 index 0000000000..7512447168 --- /dev/null +++ b/test/framework/sandbox/sources/toy/toy-0.0_add-bug.patch @@ -0,0 +1,10 @@ +--- a/toy-0.0.orig/toy.source 2014-03-06 18:48:16.000000000 +0100 ++++ b/toy-0.0/toy.source 2020-08-18 12:19:35.000000000 +0200 +@@ -2,6 +2,6 @@ + + int main(int argc, char* argv[]){ + +- printf("I'm a toy, and proud of it.\n"); ++ printf("I'm a toy, and proud of it.\n") + return 0; + } diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index cd9aefb26f..8235215da9 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -40,6 +40,7 @@ import sys import tempfile import textwrap +import filecmp from easybuild.tools import LooseVersion from importlib import reload from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup @@ -281,6 +282,82 @@ def test_toy_broken(self): # cleanup shutil.rmtree(tmpdir) + def test_toy_broken_copy_log_build_dir(self): + """ + Test whether log files and the build directory are copied to a permanent location + after a failed installation. + """ + toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + toy_ec_txt = read_file(toy_ec) + + test_ec_txt = re.sub( + r'toy-0\.0_fix-silly-typo-in-printf-statement\.patch', + r'toy-0.0_add-bug.patch', + toy_ec_txt + ) + test_ec = os.path.join(self.test_prefix, 'toy-0.0-buggy.eb') + write_file(test_ec, test_ec_txt) + + # set up subdirectories where stuff should go + tmpdir = os.path.join(self.test_prefix, 'tmp') + tmp_log_dir = os.path.join(self.test_prefix, 'tmp-logs') + failed_install_build_dirs_path = os.path.join(self.test_prefix, 'failed-install-build-dirs') + failed_install_logs_path = os.path.join(self.test_prefix, 'failed-install-logs') + + extra_args = [ + f'--failed-install-build-dirs-path={failed_install_build_dirs_path}', + f'--failed-install-logs-path={failed_install_logs_path}', + f'--tmp-logdir={tmp_log_dir}', + ] + with self.mocked_stdout_stderr(): + outtxt = self._test_toy_build(ec_file=test_ec, extra_args=extra_args, tmpdir=tmpdir, + verify=False, fails=True, verbose=False) + + # find path to temporary log file + log_files = glob.glob(os.path.join(tmp_log_dir, '*.log')) + self.assertTrue(len(log_files) == 1, f"Expected exactly one log file, found {len(log_files)}: {log_files}") + log_file = log_files[0] + + # check that log files were copied + saved_log_files = glob.glob(os.path.join(failed_install_logs_path, log_file)) + self.assertTrue(len(saved_log_files) == 1, f"Unique copy of log file '{log_file}' made") + saved_log_file = saved_log_files[0] + self.assertTrue(filecmp.cmp(log_file, saved_log_file, shallow=False), + f"Log file '{log_file}' copied successfully") + + # check that build directories were copied + build_dir = self.test_buildpath + topdir = failed_install_build_dirs_path + + app_build_dir = os.path.join(build_dir, 'toy', '0.0', 'system-system') + # pattern: -- + subdir_pattern = '????????-??????-?????' + + # find path to toy.c + toy_c_files = glob.glob(os.path.join(app_build_dir, '**', 'toy.c')) + self.assertTrue(len(toy_c_files) == 1, f"Exactly one toy.c file found: {toy_c_files}") + toy_c_file = toy_c_files[0] + + path = os.path.join(topdir, 'toy-0.0', subdir_pattern, 'toy-0.0', os.path.basename(toy_c_file)) + res = glob.glob(path) + self.assertTrue(len(res) == 1, f"Exactly one hit found for {path}: {res}") + copied_toy_c_file = res[0] + self.assertTrue(filecmp.cmp(toy_c_file, copied_toy_c_file, shallow=False), + f"Copy of {toy_c_file} should be found under {topdir}") + + # check whether compiler error messages are present in build log + + # compiler error because of missing semicolon at end of line, could be: + # "error: expected ; before ..." + # "error: expected ';' after expression" + output_regexs = [r"^\s*toy\.c:5:44: error: expected (;|.;.)"] + + log_txt = read_file(log_file) + for regex_pattern in output_regexs: + regex = re.compile(regex_pattern, re.M) + self.assertRegex(outtxt, regex) + self.assertRegex(log_txt, regex) + def test_toy_tweaked(self): """Test toy build with tweaked easyconfig, for testing extra easyconfig parameters.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs')