Merge pull request #4601 from gkaf89/feature/error-logging

copy build directory and/or log file(s) if installation failed to path specified via `--failed-install-build-dirs-path` or `--failed-install-logs-path`

Author: boegel

easybuild/base/testing.py

@@ -117,6 +117,11 @@ def assertNotExists(self, path, msg=None):
             msg = "'%s' should not exist" % path
         self.assertFalse(os.path.exists(path), msg)
 
+    def assertAllExist(self, paths, msg=None):
+        """Assert that all paths in the given list exist"""
+        for path in paths:
+            self.assertExists(path, msg)
+
     def setUp(self):
         """Prepare test case."""
         super(TestCase, self).setUp()

easybuild/framework/easyblock.py

@@ -48,6 +48,7 @@
 import inspect
 import json
 import os
+import random
 import re
 import stat
 import sys
@@ -57,6 +58,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from enum import Enum
+from string import ascii_letters
 from textwrap import indent
 
 import easybuild.tools.environment as env
@@ -79,17 +81,19 @@
 from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES  # noqa
 from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES
 from easybuild.tools.config import MOD_SEARCH_PATH_HEADERS, PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS
-from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
+from easybuild.tools.config import build_option, build_path, get_failed_install_build_dirs_path
+from easybuild.tools.config import get_failed_install_logs_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256
 from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, check_lock
-from easybuild.tools.filetools import compute_checksum, convert_name, copy_file, create_lock, create_patch_info
-from easybuild.tools.filetools import derive_alt_pypi_url, diff_files, dir_contains_files, download_file
-from easybuild.tools.filetools import encode_class_name, extract_file, find_backup_name_candidate
-from easybuild.tools.filetools import get_cwd, get_source_tarball_from_git, is_alt_pypi_url
-from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
-from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file, symlink
+from easybuild.tools.filetools import compute_checksum, convert_name, copy_dir, copy_file, create_lock
+from easybuild.tools.filetools import create_non_existing_paths, create_patch_info, derive_alt_pypi_url, diff_files
+from easybuild.tools.filetools import dir_contains_files, download_file, encode_class_name, extract_file
+from easybuild.tools.filetools import find_backup_name_candidate, get_cwd, get_source_tarball_from_git, is_alt_pypi_url
+from easybuild.tools.filetools import is_binary, is_parent_path, is_sha256_checksum, mkdir, move_file, move_logs
+from easybuild.tools.filetools import read_file, remove_dir, remove_file, remove_lock, symlink, verify_checksum
+from easybuild.tools.filetools import weld_paths, write_file
 from easybuild.tools.hooks import (
     BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP,
     MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP,
@@ -4438,6 +4442,54 @@ def print_dry_run_note(loc, silent=True):
     dry_run_msg(msg, silent=silent)
 
 
+def copy_build_dirs_logs_failed_install(application_log, silent, app, easyconfig):
+    """
+    Copy build directories and log files for failed installation (if desired)
+    """
+    logs_path = get_failed_install_logs_path(easyconfig)
+    build_dirs_path = get_failed_install_build_dirs_path(easyconfig)
+
+    # there may be multiple log files, or the file name may be different due to zipping
+    logs = glob.glob(f"{application_log}*")
+
+    timestamp = time.strftime('%Y%m%d-%H%M%S')
+    salt = ''.join(random.choices(ascii_letters, k=5))
+    unique_subdir = f'{timestamp}-{salt}'
+
+    operation_args = []
+
+    if logs_path and logs:
+        logs_path = os.path.join(logs_path, unique_subdir)
+
+        if is_parent_path(app.builddir, logs_path):
+            msg = "Path to copy log files of failed installs to is subdirectory of build directory; not copying"
+            print_warning(msg, log=_log, silent=silent)
+        else:
+            msg = f"Log file(s) of failed installation copied to {logs_path}"
+            operation_args.append((copy_file, logs, logs_path, msg))
+
+    if build_dirs_path and os.path.isdir(app.builddir):
+        build_dirs_path = os.path.join(build_dirs_path, unique_subdir)
+
+        if is_parent_path(app.builddir, build_dirs_path):
+            msg = "Path to copy build dirs of failed installs to is subdirectory of build directory; not copying"
+            print_warning(msg, log=_log, silent=silent)
+        else:
+            msg = f"Build directory of failed installation copied to {build_dirs_path}"
+
+            def operation(src, dest):
+                copy_dir(src, dest, dirs_exist_ok=True)
+
+            operation_args.append((operation, [app.builddir], build_dirs_path, msg))
+
+    persistence_paths = create_non_existing_paths(target_path for (_, _, target_path, _) in operation_args)
+
+    for idx, (operation, paths, _, msg) in enumerate(operation_args):
+        for path in paths:
+            operation(path, persistence_paths[idx])
+        print_msg(msg, log=_log, silent=silent)
+
+
 def build_and_install_one(ecdict, init_env):
     """
     Build the software
@@ -4696,6 +4748,9 @@ def ensure_writable_log_dir(log_dir):
         logs = glob.glob('%s*' % application_log)
         print_msg("Results of the build can be found in the log file(s) %s" % ', '.join(logs), log=_log, silent=silent)
 
+    if not success:
+        copy_build_dirs_logs_failed_install(application_log, silent, app, ecdict['ec'])
+
     del app
 
     return (success, application_log, error_msg, exit_code)

easybuild/tools/config.py

@@ -505,6 +505,8 @@ class ConfigurationVariables(BaseConfigurationVariables):
         'buildpath',
         'config',
         'containerpath',
+        'failed_install_build_dirs_path',
+        'failed_install_logs_path',
         'installpath',
         'installpath_modules',
         'installpath_software',
@@ -877,6 +879,42 @@ def log_path(ec=None):
     return log_file_format(return_directory=True, ec=ec, date=date, timestamp=timestamp)
 
 
+def get_failed_install_build_dirs_path(ec):
+    """
+    Return the location where the build directory is copied to if installation failed
+
+    :param ec:  dict-like value with 'name' and 'version' keys defined
+    """
+    base_path = ConfigurationVariables()['failed_install_build_dirs_path']
+    if not base_path:
+        return None
+
+    try:
+        name, version = ec['name'], ec['version']
+    except KeyError:
+        raise EasyBuildError("The 'name' and 'version' keys are required.")
+
+    return os.path.join(base_path, f'{name}-{version}')
+
+
+def get_failed_install_logs_path(ec):
+    """
+    Return the location where log files are copied to if installation failed
+
+    :param ec:  dict-like value with 'name' and 'version' keys defined
+    """
+    base_path = ConfigurationVariables()['failed_install_logs_path']
+    if not base_path:
+        return None
+
+    try:
+        name, version = ec['name'], ec['version']
+    except KeyError:
+        raise EasyBuildError("The 'name' and 'version' keys are required.")
+
+    return os.path.join(base_path, f'{name}-{version}')
+
+
 def get_build_log_path():
     """
     Return (temporary) directory for build log

easybuild/tools/filetools.py

@@ -603,6 +603,19 @@ def normalize_path(path):
     return start_slashes + os.path.sep.join(filtered_comps)
 
 
+def is_parent_path(path1, path2):
+    """
+    Return True if path1 is a prefix of path2
+
+    :param path1: absolute or relative path
+    :param path2: absolute or relative path
+    """
+    path1 = os.path.realpath(path1)
+    path2 = os.path.realpath(path2)
+    common_path = os.path.commonprefix([path1, path2])
+    return common_path == path1
+
+
 def is_alt_pypi_url(url):
     """Determine whether specified URL is already an alternative PyPI URL, i.e. whether it contains a hash."""
     # example: .../packages/5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf/easybuild-2.7.0.tar.gz
@@ -3096,3 +3109,69 @@ def create_unused_dir(parent_folder, name):
     set_gid_sticky_bits(path, recursive=True)
 
     return path
+
+
+def get_first_non_existing_parent_path(path):
+    """
+    Get first directory that does not exist, starting at path and going up.
+    """
+    path = os.path.abspath(path)
+
+    non_existing_parent = None
+    while not os.path.exists(path):
+        non_existing_parent = path
+        path = os.path.dirname(path)
+
+    return non_existing_parent
+
+
+def create_non_existing_paths(paths, max_tries=10000):
+    """
+    Create directories with given paths (including the parent directories).
+    When a directory in the same location for any of the specified paths already exists,
+    then the suffix '_<i>' is appended , with i iteratively picked between 0 and (max_tries-1),
+    until an index is found so that all required paths are non-existing.
+    All created directories have the same suffix.
+
+    :param paths: list of directory paths to be created
+    :param max_tries: maximum number of tries before failing
+    """
+    paths = [os.path.abspath(p) for p in paths]
+    for idx_path, path in enumerate(paths):
+        for idx_parent, parent in enumerate(paths):
+            if idx_parent != idx_path and is_parent_path(parent, path):
+                raise EasyBuildError(f"Path '{parent}' is a parent path of '{path}'.")
+
+    first_non_existing_parent_paths = [get_first_non_existing_parent_path(p) for p in paths]
+
+    non_existing_paths = paths
+    all_paths_created = False
+    suffix = -1
+    while suffix < max_tries and not all_paths_created:
+        tried_paths = []
+        if suffix >= 0:
+            non_existing_paths = [f'{p}_{suffix}' for p in paths]
+        try:
+            for path in non_existing_paths:
+                tried_paths.append(path)
+                # os.makedirs will raise OSError if directory already exists
+                os.makedirs(path)
+            all_paths_created = True
+        except OSError as err:
+            # Distinguish between error due to existing folder and anything else
+            if not os.path.exists(tried_paths[-1]):
+                raise EasyBuildError("Failed to create directory %s: %s", tried_paths[-1], err)
+            remove(tried_paths[:-1])
+        except BaseException as err:
+            remove(tried_paths)
+            raise err
+        suffix += 1
+
+    if not all_paths_created:
+        raise EasyBuildError(f"Exceeded maximum number of attempts ({max_tries}) to generate non-existing paths")
+
+    # set group ID and sticky bits, if desired
+    for path in first_non_existing_parent_paths:
+        set_gid_sticky_bits(path, recursive=True)
+
+    return non_existing_paths

easybuild/tools/options.py

@@ -87,7 +87,7 @@
 from easybuild.tools.docs import list_easyblocks, list_toolchains
 from easybuild.tools.environment import restore_env, unset_env_vars
 from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, get_cwd
-from easybuild.tools.filetools import install_fake_vsc, move_file, which
+from easybuild.tools.filetools import install_fake_vsc, move_file, which, is_parent_path
 from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED
 from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS
 from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS
@@ -592,6 +592,12 @@ def config_options(self):
                                      [DEFAULT_ENVVAR_USERS_MODULES]),
             'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata "
                                           "for external modules (INI format)", 'strlist', 'store', None),
+            'failed-install-build-dirs-path': ("Location where build directories are copied if installation fails; "
+                                               "an empty value disables copying of build directories",
+                                               None, 'store', None, {'metavar': "PATH"}),
+            'failed-install-logs-path': ("Location where log files are copied if installation fails; "
+                                         "an empty value disables copying of log files",
+                                         None, 'store', None, {'metavar': "PATH"}),
             'hooks': ("Location of Python module with hook implementations", 'str', 'store', None),
             'ignore-dirs': ("Directory names to ignore when searching for files/dirs",
                             'strlist', 'store', ['.git', '.svn']),
@@ -1217,22 +1223,23 @@ def _postprocess_config(self):
         # to avoid incorrect paths being used when EasyBuild changes the current working directory
         # (see https://github.com/easybuilders/easybuild-framework/issues/3619);
         # ensuring absolute paths for 'robot' is handled separately below,
-        # because we need to be careful with the argument pass to --robot;
+        # because we need to be careful with the argument passed to --robot;
         # note: repositorypath is purposely not listed here, because it's a special case:
         # - the value could consist of a 2-tuple (<path>, <relative_subdir>);
         # - the <path> could also specify the location of a *remote* (Git( repository,
         #   which can be done in variety of formats (git@<url>:<org>/<repo>), https://<url>, etc.)
         #   (see also https://github.com/easybuilders/easybuild-framework/issues/3892);
-        path_opt_names = ['buildpath', 'containerpath', 'git_working_dirs_path', 'installpath',
-                          'installpath_modules', 'installpath_software', 'prefix', 'packagepath',
-                          'robot_paths', 'sourcepath']
+        path_opt_names = ['buildpath', 'containerpath', 'failed_install_build_dirs_path', 'failed_install_logs_path',
+                          'git_working_dirs_path', 'installpath', 'installpath_modules', 'installpath_software',
+                          'prefix', 'packagepath', 'robot_paths', 'sourcepath']
 
         for opt_name in path_opt_names:
             self._ensure_abs_path(opt_name)
 
         if self.options.prefix is not None:
-            # prefix applies to all paths, and repository has to be reinitialised to take new repositorypath in account
-            # in the legacy-style configuration, repository is initialised in configuration file itself
+            # prefix applies to selected path configuration options;
+            # repository has to be reinitialised to take new repositorypath in account;
+            # in the legacy-style configuration, repository is initialised in configuration file itself;
             path_opts = ['buildpath', 'containerpath', 'installpath', 'packagepath', 'repository', 'repositorypath',
                          'sourcepath']
             for dest in path_opts:
@@ -1293,6 +1300,20 @@ def _postprocess_config(self):
         if self.options.inject_checksums or self.options.inject_checksums_to_json:
             self.options.pre_create_installdir = False
 
+        # Prevent that build directories and logs for failed installations are copied to location for build directories
+        if self.options.buildpath and self.options.failed_install_logs_path:
+            if is_parent_path(self.options.buildpath, self.options.failed_install_logs_path):
+                raise EasyBuildError(
+                    f"The --failed-install-logs-path ('{self.options.failed_install_logs_path}') "
+                    f"cannot reside in a subdirectory of the --buildpath ('{self.options.buildpath}')"
+                )
+        if self.options.buildpath and self.options.failed_install_build_dirs_path:
+            if is_parent_path(self.options.buildpath, self.options.failed_install_build_dirs_path):
+                raise EasyBuildError(
+                    f"The --failed-install-build-dirs-path ('{self.options.failed_install_build_dirs_path}') "
+                    f"cannot reside in a subdirectory of the --buildpath ('{self.options.buildpath}')"
+                )
+
     def _postprocess_list_avail(self):
         """Create all the additional info that can be requested (exit at the end)"""
         msg = ''