diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index 2a3260ae80..56a64c5588 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -53,8 +53,9 @@ from easybuild.tools.build_log import EasyBuildError, print_msg, print_warning from easybuild.tools.config import build_option from easybuild.tools.environment import restore_env -from easybuild.tools.filetools import find_easyconfigs, is_patch_file, read_file, resolve_path, which, write_file -from easybuild.tools.github import fetch_easyconfigs_from_pr, download_repo +from easybuild.tools.filetools import find_easyconfigs, is_patch_file, locate_files +from easybuild.tools.filetools import read_file, resolve_path, which, write_file +from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_files_from_pr, download_repo from easybuild.tools.multidiff import multidiff from easybuild.tools.py2vs3 import OrderedDict from easybuild.tools.toolchain.toolchain import is_system_toolchain @@ -348,44 +349,13 @@ def det_easyconfig_paths(orig_paths): ec_files = [path for path in pr_files if path.endswith('.eb')] if ec_files and robot_path: - # look for easyconfigs with relative paths in robot search path, - # unless they were found at the given relative paths - - # determine which easyconfigs files need to be found, if any - ecs_to_find = [] - for idx, ec_file in enumerate(ec_files): - if ec_file == os.path.basename(ec_file) and not os.path.exists(ec_file): - ecs_to_find.append((idx, ec_file)) - _log.debug("List of easyconfig files to find: %s" % ecs_to_find) - - # find missing easyconfigs by walking paths in robot search path - for path in robot_path: - _log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path)) - for (subpath, dirnames, filenames) in os.walk(path, topdown=True): - for idx, orig_path in ecs_to_find[:]: - if orig_path in filenames: - full_path = os.path.join(subpath, orig_path) - _log.info("Found %s in %s: %s" % (orig_path, path, full_path)) - ec_files[idx] = full_path - # if file was found, stop looking for it (first hit wins) - ecs_to_find.remove((idx, orig_path)) - - # stop os.walk insanity as soon as we have all we need (os.walk loop) - if not ecs_to_find: - break - - # ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk - dirnames[:] = [d for d in dirnames if d not in build_option('ignore_dirs')] - - # ignore archived easyconfigs, unless specified otherwise - if not build_option('consider_archived_easyconfigs'): - dirnames[:] = [d for d in dirnames if d != EASYCONFIGS_ARCHIVE_DIR] - - # stop os.walk insanity as soon as we have all we need (outer loop) - if not ecs_to_find: - break - - return [os.path.abspath(ec_file) for ec_file in ec_files] + ignore_subdirs = build_option('ignore_dirs') + if not build_option('consider_archived_easyconfigs'): + ignore_subdirs.append(EASYCONFIGS_ARCHIVE_DIR) + + ec_files = locate_files(ec_files, robot_path, ignore_subdirs=ignore_subdirs) + + return ec_files def parse_easyconfigs(paths, validate=True): @@ -728,3 +698,65 @@ def avail_easyblocks(): easyblock_mod_name, easyblocks[easyblock_mod_name]['loc'], path) return easyblocks + + +def det_copy_ec_specs(orig_paths, from_pr): + """Determine list of paths + target directory for --copy-ec.""" + + target_path, paths = None, [] + + # if only one argument is specified, use current directory as target directory + if len(orig_paths) == 1: + target_path = os.getcwd() + paths = orig_paths[:] + + # if multiple arguments are specified, assume that last argument is target location, + # and remove that from list of paths to copy + elif orig_paths: + target_path = orig_paths[-1] + paths = orig_paths[:-1] + + # if --from-pr was used in combination with --copy-ec, some extra care must be taken + if from_pr: + # pull in the paths to all the changed files in the PR, + # which includes easyconfigs but also patch files (& maybe more); + # do this in a dedicated subdirectory of the working tmpdir, + # to avoid potential trouble with already existing files in the working tmpdir + # (note: we use a fixed subdirectory in the working tmpdir here rather than a unique random subdirectory, + # to ensure that the caching for fetch_files_from_pr works across calls for the same PR) + tmpdir = os.path.join(tempfile.gettempdir(), 'fetch_files_from_pr_%s' % from_pr) + pr_paths = fetch_files_from_pr(pr=from_pr, path=tmpdir) + + # assume that files need to be copied to current working directory for now + target_path = os.getcwd() + + if orig_paths: + last_path = orig_paths[-1] + + # check files touched by PR and see if the target directory for --copy-ec + # corresponds to the name of one of these files; + # if so we should copy the specified file(s) to the current working directory, + # since interpreting the last argument as target location is very unlikely to be correct in this case + pr_filenames = [os.path.basename(p) for p in pr_paths] + if last_path in pr_filenames: + paths = orig_paths[:] + else: + target_path = last_path + # exclude last argument that is used as target location + paths = orig_paths[:-1] + + # if list of files to copy is empty at this point, + # we simply copy *all* files touched by the PR + if not paths: + paths = pr_paths + + # replace path for files touched by PR (no need to worry about others) + for idx, path in enumerate(paths): + filename = os.path.basename(path) + pr_matches = [x for x in pr_paths if os.path.basename(x) == filename] + if len(pr_matches) == 1: + paths[idx] = pr_matches[0] + elif pr_matches: + raise EasyBuildError("Found multiple paths for %s in PR: %s", filename, pr_matches) + + return paths, target_path diff --git a/easybuild/main.py b/easybuild/main.py index bd7a0b0f5e..616aca23cc 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -50,17 +50,18 @@ from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check -from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph +from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph, det_copy_ec_specs from easybuild.framework.easyconfig.tools import det_easyconfig_paths, dump_env_script, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs, review_pr, run_contrib_checks, skip_available from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option from easybuild.tools.containers.common import containerize from easybuild.tools.docs import list_software -from easybuild.tools.filetools import adjust_permissions, cleanup, copy_file, copy_files, dump_index, load_index -from easybuild.tools.filetools import read_file, register_lock_cleanup_signal_handlers, write_file -from easybuild.tools.github import check_github, close_pr, new_branch_github, find_easybuild_easyconfig -from easybuild.tools.github import install_github_token, list_prs, new_pr, new_pr_from_branch, merge_pr +from easybuild.tools.filetools import adjust_permissions, cleanup, copy_files, dump_index, load_index +from easybuild.tools.filetools import locate_files, read_file, register_lock_cleanup_signal_handlers, write_file +from easybuild.tools.github import check_github, close_pr, find_easybuild_easyconfig +from easybuild.tools.github import install_github_token, list_prs, merge_pr, new_branch_github, new_pr +from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr from easybuild.tools.hooks import START, END, load_hooks, run_hook from easybuild.tools.modules import modules_tool @@ -303,12 +304,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) - if len(orig_paths) == 1: - # if only one easyconfig file is specified, use current directory as target directory - target_path = os.getcwd() - elif orig_paths: - # last path is target when --copy-ec is used, so remove that from the list - target_path = orig_paths.pop() if options.copy_ec else None + if options.copy_ec: + # figure out list of files to copy + target location (taking into account --from-pr) + orig_paths, target_path = det_copy_ec_specs(orig_paths, options.from_pr) categorized_paths = categorize_files_by_type(orig_paths) @@ -321,17 +319,17 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: + # only copy easyconfigs here if we're not using --try-* (that's handled below) + copy_ec = options.copy_ec and not tweaked_ecs_paths + + if copy_ec or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: - if len(determined_paths) == 1: - copy_file(determined_paths[0], target_path) - print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=False) - elif len(determined_paths) > 1: - copy_files(determined_paths, target_path) - print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False) - else: - raise EasyBuildError("One of more files to copy should be specified!") + # at this point some paths may still just be filenames rather than absolute paths, + # so try to determine full path for those too via robot search path + paths = locate_files(orig_paths, robot_path) + + copy_files(paths, target_path, target_single_file=True, allow_empty=False, verbose=True) elif options.fix_deprecated_easyconfigs: fix_deprecated_easyconfigs(determined_paths) @@ -361,7 +359,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path - regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) + regtest_ok = regtest([x for (x, _) in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 @@ -429,8 +427,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): if tweaked_ecs_in_all_ecs: # Clean them, then copy them clean_up_easyconfigs(tweaked_ecs_in_all_ecs) - copy_files(tweaked_ecs_in_all_ecs, target_path) - print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False) + copy_files(tweaked_ecs_in_all_ecs, target_path, allow_empty=False, verbose=True) + + clean_exit(logfile, eb_tmpdir, testing) # creating/updating PRs if pr_options: diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 10d5a19add..da0e35ab4e 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -101,6 +101,7 @@ DEFAULT_PKG_TOOL = PKG_TOOL_FPM DEFAULT_PKG_TYPE = PKG_TYPE_RPM DEFAULT_PNS = 'EasyBuildPNS' +DEFAULT_PR_TARGET_ACCOUNT = 'easybuilders' DEFAULT_PREFIX = os.path.join(os.path.expanduser('~'), ".local", "easybuild") DEFAULT_REPOSITORY = 'FileRepository' DEFAULT_WAIT_ON_LOCK_INTERVAL = 60 @@ -204,7 +205,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'pr_branch_name', 'pr_commit_msg', 'pr_descr', - 'pr_target_account', 'pr_target_repo', 'pr_title', 'rpath_filter', @@ -307,6 +307,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_PKG_TYPE: [ 'package_type', ], + DEFAULT_PR_TARGET_ACCOUNT: [ + 'pr_target_account', + ], GENERAL_CLASS: [ 'suffix_modules_path', ], diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 38640056cf..c800b7de5d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -64,11 +64,11 @@ from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL -from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_REPOSITORY -from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT, EBROOT_ENV_VAR_ACTIONS -from easybuild.tools.config import ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE, JOB_DEPS_TYPE_ABORT_ON_ERROR -from easybuild.tools.config import JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS, LOCAL_VAR_NAMING_CHECK_WARN -from easybuild.tools.config import LOCAL_VAR_NAMING_CHECKS, WARN +from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_PR_TARGET_ACCOUNT +from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT +from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, ERROR, FORCE_DOWNLOAD_CHOICES, GENERAL_CLASS, IGNORE +from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, LOADED_MODULES_ACTIONS +from easybuild.tools.config import LOCAL_VAR_NAMING_CHECK_WARN, LOCAL_VAR_NAMING_CHECKS, WARN from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.configobj import ConfigObj, ConfigObjError from easybuild.tools.docs import FORMAT_TXT, FORMAT_RST @@ -78,7 +78,7 @@ from easybuild.tools.environment import restore_env, unset_env_vars from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, CHECKSUM_TYPES, expand_glob_paths, install_fake_vsc from easybuild.tools.filetools import move_file, which -from easybuild.tools.github import GITHUB_EB_MAIN, GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED +from easybuild.tools.github import GITHUB_PR_DIRECTION_DESC, GITHUB_PR_ORDER_CREATED from easybuild.tools.github import GITHUB_PR_STATE_OPEN, GITHUB_PR_STATES, GITHUB_PR_ORDERS, GITHUB_PR_DIRECTIONS from easybuild.tools.github import HAVE_GITHUB_API, HAVE_KEYRING, VALID_CLOSE_PR_REASONS from easybuild.tools.github import fetch_easyblocks_from_pr, fetch_github_token @@ -642,7 +642,7 @@ def github_options(self): str, 'store', None), 'pr-commit-msg': ("Commit message for new/updated pull request created with --new-pr", str, 'store', None), 'pr-descr': ("Description for new pull request created with --new-pr", str, 'store', None), - 'pr-target-account': ("Target account for new PRs", str, 'store', GITHUB_EB_MAIN), + 'pr-target-account': ("Target account for new PRs", str, 'store', DEFAULT_PR_TARGET_ACCOUNT), 'pr-target-branch': ("Target branch for new PRs", str, 'store', DEFAULT_BRANCH), 'pr-target-repo': ("Target repository for new/updating PRs (default: auto-detect based on provided files)", str, 'store', None), diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c36ea1ee64..2e278bbd32 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -56,7 +56,8 @@ from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str from easybuild.framework.easyconfig.style import check_easyconfigs_style from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph -from easybuild.framework.easyconfig.tools import find_related_easyconfigs, get_paths_for, parse_easyconfigs +from easybuild.framework.easyconfig.tools import det_copy_ec_specs, find_related_easyconfigs, get_paths_for +from easybuild.framework.easyconfig.tools import parse_easyconfigs from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one from easybuild.framework.extension import resolve_exts_filter_template from easybuild.toolchains.system import SystemToolchain @@ -4023,6 +4024,84 @@ def test_cuda_compute_capabilities(self): self.assertEqual(ec['configopts'], 'sm_42,sm_63') self.assertEqual(ec['preconfigopts'], 'sm_42 sm_63') + def test_det_copy_ec_specs(self): + """Test det_copy_ec_specs function.""" + + cwd = os.getcwd() + + # no problems on empty list as input + paths, target_path = det_copy_ec_specs([], None) + self.assertEqual(paths, []) + self.assertEqual(target_path, None) + + # single-element list, no --from-pr => use current directory as target location + paths, target_path = det_copy_ec_specs(['test.eb'], None) + self.assertEqual(paths, ['test.eb']) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # multi-element list, no --from-pr => last element is used as target location + for args in (['test.eb', 'dir'], ['test1.eb', 'test2.eb', 'dir']): + paths, target_path = det_copy_ec_specs(args, None) + self.assertEqual(paths, args[:-1]) + self.assertEqual(target_path, args[-1]) + + # use fixed PR (speeds up the test due to caching in fetch_files_from_pr; + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/8007 + from_pr = 8007 + arrow_ec_fn = 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb' + bat_ec_fn = 'bat-0.3.3-intel-2017b-Python-3.6.3.eb' + bat_patch_fn = 'bat-0.3.3-fix-pyspark.patch' + pr_files = [ + arrow_ec_fn, + bat_ec_fn, + bat_patch_fn, + ] + + # if no paths are specified, default is to copy all files touched by PR to current working directory + paths, target_path = det_copy_ec_specs([], from_pr) + self.assertEqual(len(paths), 3) + filenames = sorted([os.path.basename(x) for x in paths]) + self.assertEqual(filenames, sorted(pr_files)) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # last argument is used as target directory, + # unless it corresponds to a file touched by PR + args = [bat_ec_fn, 'target_dir'] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 1) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertEqual(target_path, 'target_dir') + + args = [bat_ec_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 1) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + args = [arrow_ec_fn, bat_ec_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 2) + self.assertEqual(os.path.basename(paths[0]), arrow_ec_fn) + self.assertEqual(os.path.basename(paths[1]), bat_ec_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + args = [bat_ec_fn, bat_patch_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 2) + self.assertEqual(os.path.basename(paths[0]), bat_ec_fn) + self.assertEqual(os.path.basename(paths[1]), bat_patch_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + + # also test with combination of local files and files from PR + args = [arrow_ec_fn, 'test.eb', 'test.patch', bat_patch_fn] + paths, target_path = det_copy_ec_specs(args, from_pr) + self.assertEqual(len(paths), 4) + self.assertEqual(os.path.basename(paths[0]), arrow_ec_fn) + self.assertEqual(paths[1], 'test.eb') + self.assertEqual(paths[2], 'test.patch') + self.assertEqual(os.path.basename(paths[3]), bat_patch_fn) + self.assertTrue(os.path.samefile(target_path, cwd)) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/options.py b/test/framework/options.py index 2727baa392..620be17c34 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -50,8 +50,8 @@ from easybuild.tools.config import DEFAULT_MODULECLASSES from easybuild.tools.config import find_last_log, get_build_log_path, get_module_syntax, module_classes from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, mkdir, read_file -from easybuild.tools.filetools import remove_dir, remove_file, which, write_file +from easybuild.tools.filetools import change_dir, copy_dir, copy_file, download_file, is_patch_file, mkdir +from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.github import GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO from easybuild.tools.github import URL_SEPARATOR, fetch_github_token from easybuild.tools.modules import Lmod @@ -965,19 +965,20 @@ def test_show_ec(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + def mocked_main(self, args): + """Run eb_main with mocked stdout/stderr.""" + self.mock_stderr(True) + self.mock_stdout(True) + self.eb_main(args, raise_error=True) + stderr, stdout = self.get_stderr(), self.get_stdout() + self.mock_stderr(False) + self.mock_stdout(False) + self.assertEqual(stderr, '') + return stdout.strip() + def test_copy_ec(self): """Test --copy-ec.""" - def mocked_main(args): - self.mock_stderr(True) - self.mock_stdout(True) - self.eb_main(args, raise_error=True) - stderr, stdout = self.get_stderr(), self.get_stdout() - self.mock_stderr(False) - self.mock_stdout(False) - self.assertEqual(stderr, '') - return stdout.strip() - topdir = os.path.dirname(os.path.abspath(__file__)) test_easyconfigs_dir = os.path.join(topdir, 'easyconfigs', 'test_ecs') @@ -987,8 +988,9 @@ def mocked_main(args): # basic test: copying one easyconfig file to a non-existing absolute path test_ec = os.path.join(self.test_prefix, 'test.eb') args = ['--copy-ec', 'toy-0.0.eb', test_ec] - stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_ec) + stdout = self.mocked_main(args) + regex = re.compile(r'.*/toy-0.0.eb copied to %s' % test_ec) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) self.assertTrue(os.path.exists(test_ec)) self.assertEqual(toy_ec_txt, read_file(test_ec)) @@ -1001,8 +1003,9 @@ def mocked_main(args): self.assertFalse(os.path.exists(target_fn)) args = ['--copy-ec', 'toy-0.0.eb', target_fn] - stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to test.eb') + stdout = self.mocked_main(args) + regex = re.compile(r'.*/toy-0.0.eb copied to test.eb') + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) change_dir(cwd) @@ -1013,8 +1016,9 @@ def mocked_main(args): test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') mkdir(test_target_dir) args = ['--copy-ec', 'toy-0.0.eb', test_target_dir] - stdout = mocked_main(args) - self.assertEqual(stdout, 'toy-0.0.eb copied to %s' % test_target_dir) + stdout = self.mocked_main(args) + regex = re.compile(r'.*/toy-0.0.eb copied to %s' % test_target_dir) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_ec = os.path.join(test_target_dir, 'toy-0.0.eb') self.assertTrue(os.path.exists(copied_toy_ec)) @@ -1035,7 +1039,7 @@ def check_copied_files(): # copying multiple easyconfig files to a non-existing target directory (which is created automatically) args = ['--copy-ec', 'toy-0.0.eb', 'bzip2-1.0.6-GCC-4.9.2.eb', test_target_dir] - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to %s' % test_target_dir) check_copied_files() @@ -1047,12 +1051,12 @@ def check_copied_files(): args[-1] = os.path.basename(test_target_dir) self.assertFalse(os.path.exists(args[-1])) - stdout = mocked_main(args) + stdout = self.mocked_main(args) self.assertEqual(stdout, '2 file(s) copied to test_target_dir') check_copied_files() - # copying multiple easyconfig to an existing target file resuts in an error + # copying multiple easyconfig to an existing target file results in an error target = os.path.join(self.test_prefix, 'test.eb') self.assertTrue(os.path.isfile(target)) args = ['--copy-ec', 'toy-0.0.eb', 'bzip2-1.0.6-GCC-4.9.2.eb', target] @@ -1065,8 +1069,8 @@ def check_copied_files(): change_dir(test_working_dir) self.assertEqual(len(os.listdir(os.getcwd())), 0) args = ['--copy-ec', 'toy-0.0.eb'] - stdout = mocked_main(args) - regex = re.compile('toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) + stdout = self.mocked_main(args) + regex = re.compile('.*/toy-0.0.eb copied to .*/%s' % os.path.basename(test_working_dir)) self.assertTrue(regex.match(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) copied_toy_cwd = os.path.join(test_working_dir, 'toy-0.0.eb') self.assertTrue(os.path.exists(copied_toy_cwd)) @@ -1074,9 +1078,128 @@ def check_copied_files(): # --copy-ec without arguments results in a proper error args = ['--copy-ec'] - error_pattern = "One of more files to copy should be specified!" + error_pattern = "One or more files to copy should be specified!" self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) + def test_copy_ec_from_pr(self): + """Test combination of --copy-ec with --from-pr.""" + if self.github_token is None: + print("Skipping test_copy_ec_from_pr, no GitHub token available?") + return + + test_working_dir = os.path.join(self.test_prefix, 'test_working_dir') + mkdir(test_working_dir) + test_target_dir = os.path.join(self.test_prefix, 'test_target_dir') + # Make sure the test target directory doesn't exist + remove_dir(test_target_dir) + + all_files_pr8007 = [ + 'Arrow-0.7.1-intel-2017b-Python-3.6.3.eb', + 'bat-0.3.3-fix-pyspark.patch', + 'bat-0.3.3-intel-2017b-Python-3.6.3.eb', + ] + + # test use of --copy-ec with --from-pr to the current working directory + cwd = change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007'] + stdout = self.mocked_main(args) + + regex = re.compile(r"3 file\(s\) copied to .*/%s" % os.path.basename(test_working_dir)) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + # check that the files exist + for pr_file in all_files_pr8007: + self.assertTrue(os.path.exists(os.path.join(test_working_dir, pr_file))) + remove_file(os.path.join(test_working_dir, pr_file)) + + # copying all files touched by PR to a non-existing target directory (which is created automatically) + self.assertFalse(os.path.exists(test_target_dir)) + args = ['--copy-ec', '--from-pr', '8007', test_target_dir] + stdout = self.mocked_main(args) + + regex = re.compile(r"3 file\(s\) copied to .*/%s" % os.path.basename(test_target_dir)) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + for pr_file in all_files_pr8007: + self.assertTrue(os.path.exists(os.path.join(test_target_dir, pr_file))) + remove_dir(test_target_dir) + + # test where we select a single easyconfig file from a PR + mkdir(test_target_dir) + ec_filename = 'bat-0.3.3-intel-2017b-Python-3.6.3.eb' + args = ['--copy-ec', '--from-pr', '8007', ec_filename, test_target_dir] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (ec_filename, os.path.basename(test_target_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(os.listdir(test_target_dir), [ec_filename]) + self.assertTrue("name = 'bat'" in read_file(os.path.join(test_target_dir, ec_filename))) + remove_dir(test_target_dir) + + # test copying of a single easyconfig file from a PR to a non-existing path + bat_ec = os.path.join(self.test_prefix, 'bat.eb') + args[-1] = bat_ec + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/bat.eb" % ec_filename) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertTrue(os.path.exists(bat_ec)) + self.assertTrue("name = 'bat'" in read_file(bat_ec)) + + change_dir(cwd) + remove_dir(test_working_dir) + mkdir(test_working_dir) + change_dir(test_working_dir) + + # test copying of a patch file from a PR via --copy-ec to current directory + patch_fn = 'bat-0.3.3-fix-pyspark.patch' + args = ['--copy-ec', '--from-pr', '8007', patch_fn, '.'] + stdout = self.mocked_main(args) + + self.assertEqual(os.listdir(test_working_dir), [patch_fn]) + patch_path = os.path.join(test_working_dir, patch_fn) + self.assertTrue(os.path.exists(patch_path)) + self.assertTrue(is_patch_file(patch_path)) + remove_file(patch_path) + + # test the same thing but where we don't provide a target location + change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007', ec_filename] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (ec_filename, os.path.basename(test_working_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(os.listdir(test_working_dir), [ec_filename]) + self.assertTrue("name = 'bat'" in read_file(os.path.join(test_working_dir, ec_filename))) + + # also test copying of patch file to current directory (without specifying target location) + change_dir(test_working_dir) + args = ['--copy-ec', '--from-pr', '8007', patch_fn] + stdout = self.mocked_main(args) + + regex = re.compile(r"%s copied to .*/%s" % (patch_fn, os.path.basename(test_working_dir))) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + self.assertEqual(sorted(os.listdir(test_working_dir)), sorted([ec_filename, patch_fn])) + self.assertTrue(is_patch_file(os.path.join(test_working_dir, patch_fn))) + + change_dir(cwd) + remove_dir(test_working_dir) + + # test with only one ec in the PR (final argument is taken as a filename) + test_ec = os.path.join(self.test_prefix, 'test.eb') + args = ['--copy-ec', '--from-pr', '11521', test_ec] + ec_pr11521 = "ExifTool-12.00-GCCcore-9.3.0.eb" + stdout = self.mocked_main(args) + regex = re.compile(r'.*/%s copied to %s' % (ec_pr11521, test_ec)) + self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + self.assertTrue(os.path.exists(test_ec)) + self.assertTrue("name = 'ExifTool'" in read_file(test_ec)) + remove_file(test_ec) + def test_dry_run(self): """Test dry run (long format).""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -2519,7 +2642,8 @@ def test_robot_path_check(self): # different error when a non-existing easyconfig file is specified to --robot args = ['--dry-run', '--robot', 'no_such_easyconfig_file_in_robot_search_path.eb'] - self.assertErrorRegex(EasyBuildError, "Can't find path", self.eb_main, args, raise_error=True) + error_pattern = "One or more files not found: no_such_easyconfig_file_in_robot_search_path.eb" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) for robot in ['-r%s' % self.test_prefix, '--robot=%s' % self.test_prefix]: args = ['toy-0.0.eb', '--dry-run', robot] diff --git a/test/framework/robot.py b/test/framework/robot.py index e50f570a37..dc540674ec 100644 --- a/test/framework/robot.py +++ b/test/framework/robot.py @@ -620,12 +620,19 @@ def test_det_easyconfig_paths(self): test_ec = 'toy-0.0-deps.eb' shutil.copy2(os.path.join(test_ecs_path, 't', 'toy', test_ec), self.test_prefix) + # copy hwloc easyconfig to h/hwloc subdir in robot search path, + # to trigger bug fixed in det_easyconfig_paths (.extend rather than .append for '__archive'__ to ignore_subdirs) + hwloc_ec = 'hwloc-1.11.8-GCC-6.4.0-2.28.eb' + subdir_hwloc = os.path.join(self.test_prefix, 'h', 'hwloc') + mkdir(subdir_hwloc, parents=True) + shutil.copy2(os.path.join(test_ecs_path, 'h', 'hwloc', hwloc_ec), subdir_hwloc) shutil.copy2(os.path.join(test_ecs_path, 'i', 'intel', 'intel-2018a.eb'), self.test_prefix) self.assertFalse(os.path.exists(test_ec)) args = [ os.path.join(test_ecs_path, 't', 'toy', 'toy-0.0.eb'), test_ec, # relative path, should be resolved via robot search path + hwloc_ec, '--dry-run', '--debug', '--robot', @@ -640,6 +647,7 @@ def test_det_easyconfig_paths(self): (test_ecs_path, 'toy/0.0'), # specified easyconfigs, available at given location (self.test_prefix, 'intel/2018a'), # dependency, found in robot search path (self.test_prefix, 'toy/0.0-deps'), # specified easyconfig, found in robot search path + (self.test_prefix, 'hwloc/1.11.8-GCC-6.4.0-2.28'), # specified easyconfig, found in robot search path ] for path_prefix, module in modules: ec_fn = "%s.eb" % '-'.join(module.split('/')) @@ -654,7 +662,8 @@ def test_det_easyconfig_paths(self): '--robot', '--unittest-file=%s' % self.logfile, ] - self.assertErrorRegex(EasyBuildError, "Can't find", self.eb_main, args, logfile=dummylogfn, raise_error=True) + error_pattern = "One or more files not found: intel-2012a.eb" + self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, logfile=dummylogfn, raise_error=True) args.append('--consider-archived-easyconfigs') outtxt = self.eb_main(args, logfile=dummylogfn, raise_error=True)