diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index f42d3ca4c4..fc639432e9 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -71,12 +71,12 @@ from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256 -from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file -from easybuild.tools.filetools import change_dir, convert_name, compute_checksum, copy_file, derive_alt_pypi_url -from easybuild.tools.filetools import diff_files, download_file, encode_class_name, extract_file +from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name +from easybuild.tools.filetools import compute_checksum, copy_file, check_lock, create_lock, derive_alt_pypi_url +from easybuild.tools.filetools import diff_files, dir_contains_files, download_file, encode_class_name, extract_file from easybuild.tools.filetools import find_backup_name_candidate, get_source_tarball_from_git, is_alt_pypi_url from easybuild.tools.filetools import is_binary, is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir -from easybuild.tools.filetools import remove_file, verify_checksum, weld_paths, write_file, dir_contains_files +from easybuild.tools.filetools import remove_file, remove_lock, verify_checksum, weld_paths, write_file from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP @@ -3094,30 +3094,14 @@ def run_all_steps(self, run_test_cases): if ignore_locks: self.log.info("Ignoring locks...") else: - locks_dir = build_option('locks_dir') or os.path.join(install_path('software'), '.locks') - lock_path = os.path.join(locks_dir, '%s.lock' % self.installdir.replace('/', '_')) - - # if lock already exists, either abort or wait until it disappears - if os.path.exists(lock_path): - wait_on_lock = build_option('wait_on_lock') - if wait_on_lock: - while os.path.exists(lock_path): - print_msg("lock %s exists, waiting %d seconds..." % (lock_path, wait_on_lock), - silent=self.silent) - time.sleep(wait_on_lock) - else: - raise EasyBuildError("Lock %s already exists, aborting!", lock_path) + lock_name = self.installdir.replace('/', '_') - # create lock to avoid that another installation running in parallel messes things up; - # we use a directory as a lock, since that's atomically created - try: - mkdir(lock_path, parents=True) - except EasyBuildError as err: - # clean up the error message a bit, get rid of the "Failed to create directory" part + quotes - stripped_err = str(err).split(':', 1)[1].strip().replace("'", '').replace('"', '') - raise EasyBuildError("Failed to create lock %s: %s", lock_path, stripped_err) + # check if lock already exists; + # either aborts with an error or waits until it disappears (depends on --wait-on-lock) + check_lock(lock_name) - self.log.info("Lock created: %s", lock_path) + # create lock to avoid that another installation running in parallel messes things up + create_lock(lock_name) try: for (step_name, descr, step_methods, skippable) in steps: @@ -3135,8 +3119,7 @@ def run_all_steps(self, run_test_cases): pass finally: if not ignore_locks: - remove_dir(lock_path) - self.log.info("Lock removed: %s", lock_path) + remove_lock(lock_name) # return True for successfull build (or stopped build) return True diff --git a/easybuild/main.py b/easybuild/main.py index 415321dc9a..94ed2ea301 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -57,7 +57,7 @@ from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, dump_index, load_index -from easybuild.tools.filetools import read_file, write_file +from easybuild.tools.filetools import read_file, register_lock_cleanup_signal_handlers, write_file from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr @@ -189,6 +189,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ + + register_lock_cleanup_signal_handlers() + # if $CDPATH is set, unset it, it'll only cause trouble... # see https://github.com/easybuilders/easybuild-framework/issues/2944 if 'CDPATH' in os.environ: @@ -518,5 +521,5 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): main() except EasyBuildError as err: print_error(err.msg) - except KeyboardInterrupt: - print_error("Cancelled by user (keyboard interrupt)") + except KeyboardInterrupt as err: + print_error("Cancelled by user: %s" % err) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 0bcf31ab8b..3bca0194b7 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -102,6 +102,8 @@ DEFAULT_PNS = 'EasyBuildPNS' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' +DEFAULT_WAIT_ON_LOCK_INTERVAL = 60 +DEFAULT_WAIT_ON_LOCK_LIMIT = 0 EBROOT_ENV_VAR_ACTIONS = [ERROR, IGNORE, UNSET, WARN] LOADED_MODULES_ACTIONS = [ERROR, IGNORE, PURGE, UNLOAD, WARN] @@ -211,6 +213,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'subdir_user_modules', 'test_report_env_filter', 'testoutput', + 'wait_on_lock', 'umask', 'zip_logs', ], @@ -256,7 +259,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'use_f90cache', 'use_existing_modules', 'set_default_module', - 'wait_on_lock', + 'wait_on_lock_limit', ], True: [ 'cleanup_builddir', @@ -305,6 +308,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_ALLOW_LOADED_MODULES: [ 'allow_loaded_modules', ], + DEFAULT_WAIT_ON_LOCK_INTERVAL: [ + 'wait_on_lock_interval', + ], } # build option that do not have a perfectly matching command line option BUILD_OPTIONS_OTHER = { diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index b7149cc553..c216da85e5 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -48,6 +48,7 @@ import os import re import shutil +import signal import stat import sys import tempfile @@ -59,7 +60,7 @@ from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning -from easybuild.tools.config import GENERIC_EASYBLOCK_PKG, build_option +from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, GENERIC_EASYBLOCK_PKG, build_option, install_path from easybuild.tools.py2vs3 import std_urllib, string_type from easybuild.tools.utilities import nub, remove_unwanted_chars @@ -155,6 +156,9 @@ '.tar.z': "tar xzf %(filepath)s", } +# global set of names of locks that were created in this session +global_lock_names = set() + class ZlibChecksum(object): """ @@ -1493,6 +1497,131 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): _log.debug("Not creating existing path %s" % path) +def det_lock_path(lock_name): + """ + Determine full path for lock with specifed name. + """ + locks_dir = build_option('locks_dir') or os.path.join(install_path('software'), '.locks') + return os.path.join(locks_dir, lock_name + '.lock') + + +def create_lock(lock_name): + """Create lock with specified name.""" + + lock_path = det_lock_path(lock_name) + _log.info("Creating lock at %s...", lock_path) + try: + # we use a directory as a lock, since that's atomically created + mkdir(lock_path, parents=True) + global_lock_names.add(lock_name) + except EasyBuildError as err: + # clean up the error message a bit, get rid of the "Failed to create directory" part + quotes + stripped_err = str(err).split(':', 1)[1].strip().replace("'", '').replace('"', '') + raise EasyBuildError("Failed to create lock %s: %s", lock_path, stripped_err) + _log.info("Lock created: %s", lock_path) + + +def check_lock(lock_name): + """ + Check whether a lock with specified name already exists. + + If it exists, either wait until it's released, or raise an error + (depending on --wait-on-lock configuration option). + """ + lock_path = det_lock_path(lock_name) + if os.path.exists(lock_path): + _log.info("Lock %s exists!", lock_path) + + wait_interval = build_option('wait_on_lock_interval') + wait_limit = build_option('wait_on_lock_limit') + + # --wait-on-lock is deprecated, should use --wait-on-lock-limit and --wait-on-lock-interval instead + wait_on_lock = build_option('wait_on_lock') + if wait_on_lock is not None: + depr_msg = "Use of --wait-on-lock is deprecated, use --wait-on-lock-limit and --wait-on-lock-interval" + _log.deprecated(depr_msg, '5.0') + + # if --wait-on-lock-interval has default value and --wait-on-lock is specified too, the latter wins + # (required for backwards compatibility) + if wait_interval == DEFAULT_WAIT_ON_LOCK_INTERVAL and wait_on_lock > 0: + wait_interval = wait_on_lock + + # if --wait-on-lock-limit is not specified we need to wait indefinitely if --wait-on-lock is specified, + # since the original semantics of --wait-on-lock was that it specified the waiting time interval (no limit) + if not wait_limit: + wait_limit = -1 + + # wait limit could be zero (no waiting), -1 (no waiting limit) or non-zero value (waiting limit in seconds) + if wait_limit != 0: + wait_time = 0 + while os.path.exists(lock_path) and (wait_limit == -1 or wait_time < wait_limit): + print_msg("lock %s exists, waiting %d seconds..." % (lock_path, wait_interval), + silent=build_option('silent')) + time.sleep(wait_interval) + wait_time += wait_interval + + if os.path.exists(lock_path) and wait_limit != -1 and wait_time >= wait_limit: + error_msg = "Maximum wait time for lock %s to be released reached: %s sec >= %s sec" + raise EasyBuildError(error_msg, lock_path, wait_time, wait_limit) + else: + _log.info("Lock %s was released!", lock_path) + else: + raise EasyBuildError("Lock %s already exists, aborting!", lock_path) + else: + _log.info("Lock %s does not exist", lock_path) + + +def remove_lock(lock_name): + """ + Remove lock with specified name. + """ + lock_path = det_lock_path(lock_name) + _log.info("Removing lock %s...", lock_path) + remove_dir(lock_path) + if lock_name in global_lock_names: + global_lock_names.remove(lock_name) + _log.info("Lock removed: %s", lock_path) + + +def clean_up_locks(): + """ + Clean up all still existing locks that were created in this session. + """ + for lock_name in list(global_lock_names): + remove_lock(lock_name) + + +def clean_up_locks_signal_handler(signum, frame): + """ + Signal handler, cleans up locks & exits with received signal number. + """ + + if not build_option('silent'): + print_warning("signal received (%s), cleaning up locks (%s)..." % (signum, ', '.join(global_lock_names))) + clean_up_locks() + + # by default, a KeyboardInterrupt is raised with SIGINT, so keep doing so + if signum == signal.SIGINT: + raise KeyboardInterrupt("keyboard interrupt") + else: + sys.exit(signum) + + +def register_lock_cleanup_signal_handlers(): + """ + Register signal handler for signals that cancel the current EasyBuild session, + so we can clean up the locks that were created first. + """ + signums = [ + signal.SIGABRT, + signal.SIGINT, # Ctrl-C + signal.SIGTERM, # signal 15, soft kill (like when Slurm job is cancelled or received timeout) + signal.SIGQUIT, # kinda like Ctrl-C + ] + for signum in signums: + signal.signal(signum, clean_up_locks_signal_handler) + + def expand_glob_paths(glob_paths): """Expand specified glob paths to a list of unique non-glob paths to only files.""" paths = [] diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 2a09600f78..ebad095341 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -64,10 +64,10 @@ from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE -from easybuild.tools.config import DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY, EBROOT_ENV_VAR_ACTIONS, ERROR -from easybuild.tools.config import FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR -from easybuild.tools.config import JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS, WARN -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS +from easybuild.tools.config import DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL +from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_LIMIT, EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES +from easybuild.tools.config import GENERAL_CLASS, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN +from easybuild.tools.config import LOADED_MODULES_ACTIONS, LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST @@ -76,9 +76,8 @@ from easybuild.tools.docs import list_easyblocks, list_toolchains from easybuild.tools.environment import restore_env, unset_env_vars from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, install_fake_vsc, move_file, which -from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO -from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED, GITHUB_PR_STATE_OPEN -from easybuild.tools.github import GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS +from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED +from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token from easybuild.tools.hooks import KNOWN_HOOKS @@ -442,8 +441,15 @@ def override_options(self): None, 'store_true', False), 'verify-easyconfig-filenames': ("Verify whether filename of specified easyconfigs matches with contents", None, 'store_true', False), - 'wait-on-lock': ("Wait interval (in seconds) to use when waiting for existing lock to be removed " - "(0: implies no waiting, but exiting with an error)", int, 'store', 0), + 'wait-on-lock': ("Wait for lock to be released; 0 implies no waiting (exit with an error if the lock " + "already exists), non-zero value specified waiting interval [DEPRECATED: " + "use --wait-on-lock-interval and --wait-on-lock-limit instead]", + int, 'store_or_None', None), + 'wait-on-lock-interval': ("Wait interval (in seconds) to use when waiting for existing lock to be removed", + int, 'store', DEFAULT_WAIT_ON_LOCK_INTERVAL), + 'wait-on-lock-limit': ("Maximum amount of time (in seconds) to wait until lock is released (0 means no " + "waiting at all, exit with error; -1 means no waiting limit, keep waiting)", + int, 'store', DEFAULT_WAIT_ON_LOCK_LIMIT), 'zip-logs': ("Zip logs that are copied to install directory, using specified command", None, 'store_or_None', 'gzip'), diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 166f13c8c2..54e0fad84f 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -2455,6 +2455,97 @@ def test_copy_framework_files(self): self.assertEqual(res['new'], expected_new) + def test_locks(self): + """Tests for lock-related functions.""" + + init_config(build_options={'silent': True}) + + # make sure that global list of locks is empty when we start off + self.assertFalse(ft.global_lock_names) + + # use a realistic lock name (cfr. EasyBlock.run_all_steps) + installdir = os.path.join(self.test_installpath, 'software', 'test', '1.2.3-foss-2019b-Python-3.7.4') + lock_name = installdir.replace('/', '_') + + # det_lock_path returns full path to lock with specified name + # (used internally by create_lock, check_lock, remove_lock) + lock_path = ft.det_lock_path(lock_name) + self.assertFalse(os.path.exists(lock_path)) + + locks_dir = os.path.dirname(lock_path) + self.assertFalse(os.path.exists(locks_dir)) + + # if lock doesn't exist yet, check_lock just returns + ft.check_lock(lock_name) + + # create lock, and check whether it actually was created + ft.create_lock(lock_name) + self.assertTrue(os.path.exists(lock_path)) + + # can't use os.path.samefile until locks_dir actually exists + self.assertTrue(os.path.samefile(locks_dir, os.path.join(self.test_installpath, 'software', '.locks'))) + + self.assertEqual(os.listdir(locks_dir), [lock_name + '.lock']) + + # if lock exists, then check_lock raises an error + self.assertErrorRegex(EasyBuildError, "Lock .* already exists", ft.check_lock, lock_name) + + # remove_lock should... remove the lock + ft.remove_lock(lock_name) + self.assertFalse(os.path.exists(lock_path)) + self.assertEqual(os.listdir(locks_dir), []) + + # no harm done if remove_lock is called if lock is already gone + ft.remove_lock(lock_name) + + # check_lock just returns again after lock is removed + ft.check_lock(lock_name) + + # global list of locks should be empty at this point + self.assertFalse(ft.global_lock_names) + + # calling clean_up_locks when there are no locks should not cause trouble + ft.clean_up_locks() + + ft.create_lock(lock_name) + self.assertEqual(ft.global_lock_names, set([lock_name])) + self.assertEqual(os.listdir(locks_dir), [lock_name + '.lock']) + + ft.clean_up_locks() + self.assertFalse(ft.global_lock_names) + self.assertFalse(os.path.exists(lock_path)) + self.assertEqual(os.listdir(locks_dir), []) + + # no problem with multiple locks + lock_names = [lock_name, 'test123', 'foo@bar%baz'] + lock_paths = [os.path.join(locks_dir, x + '.lock') for x in lock_names] + for ln in lock_names: + ft.create_lock(ln) + for lp in lock_paths: + self.assertTrue(os.path.exists(lp), "Path %s should exist" % lp) + + self.assertEqual(ft.global_lock_names, set(lock_names)) + expected_locks = sorted(ln + '.lock' for ln in lock_names) + self.assertEqual(sorted(os.listdir(locks_dir)), expected_locks) + + ft.clean_up_locks() + for lp in lock_paths: + self.assertFalse(os.path.exists(lp), "Path %s should not exist" % lp) + self.assertFalse(ft.global_lock_names) + self.assertEqual(os.listdir(locks_dir), []) + + # also test signal handler that is supposed to clean up locks + ft.create_lock(lock_name) + self.assertTrue(ft.global_lock_names) + self.assertTrue(os.path.exists(lock_path)) + self.assertEqual(os.listdir(locks_dir), [lock_name + '.lock']) + + # clean_up_locks_signal_handler causes sys.exit with specified exit code + self.assertErrorRegex(SystemExit, '15', ft.clean_up_locks_signal_handler, 15, None) + self.assertFalse(ft.global_lock_names) + self.assertFalse(os.path.exists(lock_path)) + self.assertEqual(os.listdir(locks_dir), []) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 26e6dacfcd..41dfe313ea 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -142,7 +142,8 @@ def check_toy(self, installpath, outtxt, version='0.0', versionprefix='', versio self.assertTrue(os.path.exists(devel_module_path)) def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, - raise_error=False, test_report=None, versionsuffix='', testing=True): + raise_error=False, test_report=None, versionsuffix='', testing=True, + raise_systemexit=False): """Perform a toy build.""" if extra_args is None: extra_args = [] @@ -169,7 +170,7 @@ def test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True myerr = None try: outtxt = self.eb_main(args, logfile=self.dummylogfn, do_build=True, verbose=verbose, - raise_error=raise_error, testing=testing) + raise_error=raise_error, testing=testing, raise_systemexit=raise_systemexit) except Exception as err: myerr = err if raise_error: @@ -2758,43 +2759,86 @@ def __enter__(self): def __exit__(self, type, value, traceback): pass - # wait for lock to be removed, with 1 second interval of checking - extra_args.append('--wait-on-lock=1') + # wait for lock to be removed, with 1 second interval of checking; + # check with both --wait-on-lock-interval and deprecated --wait-on-lock options wait_regex = re.compile("^== lock .*_software_toy_0.0.lock exists, waiting 1 seconds", re.M) ok_regex = re.compile("^== COMPLETED: Installation ended successfully", re.M) - self.assertTrue(os.path.exists(toy_lock_path)) + test_cases = [ + ['--wait-on-lock=1'], + ['--wait-on-lock=1', '--wait-on-lock-interval=60'], + ['--wait-on-lock=100', '--wait-on-lock-interval=1'], + ['--wait-on-lock-limit=100', '--wait-on-lock=1'], + ['--wait-on-lock-limit=100', '--wait-on-lock-interval=1'], + ['--wait-on-lock-limit=-1', '--wait-on-lock=1'], + ['--wait-on-lock-limit=-1', '--wait-on-lock-interval=1'], + ] - # use context manager to remove lock after 3 seconds - with remove_lock_after(3, toy_lock_path): - self.mock_stderr(True) - self.mock_stdout(True) - self.test_toy_build(extra_args=extra_args, verify=False, raise_error=True, testing=False) - stderr, stdout = self.get_stderr(), self.get_stdout() - self.mock_stderr(False) - self.mock_stdout(False) + for opts in test_cases: - self.assertEqual(stderr, '') + if any('--wait-on-lock=' in x for x in opts): + self.allow_deprecated_behaviour() + else: + self.disallow_deprecated_behaviour() - wait_matches = wait_regex.findall(stdout) - # we can't rely on an exact number of 'waiting' messages, so let's go with a range... - self.assertTrue(len(wait_matches) in range(2, 5)) + if not os.path.exists(toy_lock_path): + mkdir(toy_lock_path) - self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) + self.assertTrue(os.path.exists(toy_lock_path)) + + all_args = extra_args + opts + + # use context manager to remove lock after 3 seconds + with remove_lock_after(3, toy_lock_path): + self.mock_stderr(True) + self.mock_stdout(True) + self.test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) - # when there is no lock in place, --wait-on-lock has no impact - self.assertFalse(os.path.exists(toy_lock_path)) + if any('--wait-on-lock=' in x for x in all_args): + self.assertTrue("Use of --wait-on-lock is deprecated" in stderr) + else: + self.assertEqual(stderr, '') + + wait_matches = wait_regex.findall(stdout) + # we can't rely on an exact number of 'waiting' messages, so let's go with a range... + self.assertTrue(len(wait_matches) in range(2, 5)) + + self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) + + # check use of --wait-on-lock-limit: if lock is never removed, we should give up when limit is reached + mkdir(toy_lock_path) + all_args = extra_args + ['--wait-on-lock-limit=3', '--wait-on-lock-interval=1'] self.mock_stderr(True) self.mock_stdout(True) - self.test_toy_build(extra_args=extra_args, verify=False, raise_error=True, testing=False) + error_pattern = r"Maximum wait time for lock /.*toy_0.0.lock to be released reached: [0-9]+ sec >= 3 sec" + self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=all_args, + verify=False, raise_error=True, testing=False) stderr, stdout = self.get_stderr(), self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) - self.assertEqual(stderr, '') - self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) - self.assertFalse(wait_regex.search(stdout), "Pattern '%s' not found in: %s" % (wait_regex.pattern, stdout)) + wait_matches = wait_regex.findall(stdout) + self.assertTrue(len(wait_matches) in range(2, 5)) + + # when there is no lock in place, --wait-on-lock* has no impact + remove_dir(toy_lock_path) + for opt in ['--wait-on-lock=1', '--wait-on-lock-limit=3', '--wait-on-lock-interval=1']: + all_args = extra_args + [opt] + self.assertFalse(os.path.exists(toy_lock_path)) + self.mock_stderr(True) + self.mock_stdout(True) + self.test_toy_build(extra_args=all_args, verify=False, raise_error=True, testing=False) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + + self.assertEqual(stderr, '') + self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) + self.assertFalse(wait_regex.search(stdout), "Pattern '%s' not found in: %s" % (wait_regex.pattern, stdout)) # check for clean error on creation of lock extra_args = ['--locks-dir=/'] @@ -2803,6 +2847,57 @@ def __exit__(self, type, value, traceback): self.assertErrorRegex(EasyBuildError, error_pattern, self.test_toy_build, extra_args=extra_args, raise_error=True, verbose=False) + def test_toy_lock_cleanup_signals(self): + """Test cleanup of locks after EasyBuild session gets a cancellation signal.""" + + locks_dir = os.path.join(self.test_installpath, 'software', '.locks') + self.assertFalse(os.path.exists(locks_dir)) + + # context manager which stops the function being called with the specified signal + class wait_and_signal: + def __init__(self, seconds, signum): + self.seconds = seconds + self.signum = signum + + def send_signal(self, *args): + os.kill(os.getpid(), self.signum) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.send_signal) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + pass + + # add extra sleep command to ensure session takes long enough + test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + toy_ec_txt = read_file(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0.eb')) + + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, toy_ec_txt + '\npostinstallcmds = ["sleep 5"]') + + signums = [ + (signal.SIGABRT, SystemExit), + (signal.SIGINT, KeyboardInterrupt), + (signal.SIGTERM, SystemExit), + (signal.SIGQUIT, SystemExit), + ] + for (signum, exc) in signums: + with wait_and_signal(1, signum): + self.mock_stderr(True) + self.mock_stdout(True) + self.assertErrorRegex(exc, '.*', self.test_toy_build, ec_file=test_ec, verify=False, + raise_error=True, testing=False, raise_systemexit=True) + + stderr = self.get_stderr().strip() + self.mock_stderr(False) + self.mock_stdout(False) + + pattern = r"^WARNING: signal received \(%s\), " % int(signum) + pattern += r"cleaning up locks \(.*software_toy_0.0\)\.\.\." + regex = re.compile(pattern) + self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) + def test_toy_build_unicode_description(self): """Test installation of easyconfig file that has non-ASCII characters in description.""" # cfr. https://github.com/easybuilders/easybuild-framework/issues/3284